diff --git a/projects/fangjia-coresite/src/main/webapp/company/about.ftl b/projects/fangjia-coresite/src/main/webapp/company/about.ftl
index 778557075..16b30ed80 100644
--- a/projects/fangjia-coresite/src/main/webapp/company/about.ftl
+++ b/projects/fangjia-coresite/src/main/webapp/company/about.ftl
@@ -158,37 +158,36 @@
<#-- -->
-
+
-
+
-
+
-
+
-
+
-
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java
new file mode 100644
index 000000000..8275f68f8
--- /dev/null
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java
@@ -0,0 +1,9 @@
+package com.fangjia.dal.mongo.open.dao;
+
+import com.fangjia.dal.MongoDao;
+import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
+
+public interface CommunityBasePriceDao extends MongoDao {
+
+}
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java
new file mode 100644
index 000000000..97f9d48de
--- /dev/null
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java
@@ -0,0 +1,9 @@
+package com.fangjia.dal.mongo.open.dao;
+
+import com.fangjia.dal.MongoDao;
+import com.fangjia.dal.mongo.model.CountDistrictStatBycity;
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
+
+public interface CommunityPoiDao extends MongoDao {
+
+}
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java
new file mode 100644
index 000000000..c89477b09
--- /dev/null
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java
@@ -0,0 +1,10 @@
+package com.fangjia.dal.mongo.open.dao.impl;
+
+import com.fangjia.dal.MongoDaoImpl;
+import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
+import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
+import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
+
+public class CommunityBasePriceDaoImpl extends MongoDaoImpl implements CommunityBasePriceDao {
+}
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java
new file mode 100644
index 000000000..e8a317b04
--- /dev/null
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java
@@ -0,0 +1,10 @@
+package com.fangjia.dal.mongo.open.dao.impl;
+
+import com.fangjia.dal.MongoDaoImpl;
+import com.fangjia.dal.mongo.open.dao.ApiVisitCountDao;
+import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
+import com.fangjia.dal.mongo.tool.model.ApiVisitCount;
+
+public class CommunityPoiDaoImpl extends MongoDaoImpl implements CommunityPoiDao {
+}
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java
new file mode 100644
index 000000000..9321bd542
--- /dev/null
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java
@@ -0,0 +1,74 @@
+package com.fangjia.dal.mongo.open.model;
+
+import com.fangjia.dal.MongoEntity;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.Table;
+
+/**
+ * 小区建行置信度信息
+ *
+ * @author shenjian@fangjia.com
+ */
+@Entity
+@Table(name = "business.community_base_price_gj")
+public class CommunityBasePrice implements MongoEntity {
+
+ private static final long serialVersionUID = -8010010773207622878L;
+
+ private String id;
+ private String city;
+ private String region;
+ private String name;
+ private Double credit;
+
+ @Id
+ @Column(name = "_id")
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ @Override
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ @Column(name = "city")
+ public String getCity() {
+ return city;
+ }
+
+ public void setCity(String city) {
+ this.city = city;
+ }
+
+ @Column(name = "region")
+ public String getRegion() {
+ return region;
+ }
+
+ public void setRegion(String region) {
+ this.region = region;
+ }
+
+ @Column(name = "name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Column(name = "credit")
+ public Double getCredit() {
+ return credit;
+ }
+
+ public void setCredit(Double credit) {
+ this.credit = credit;
+ }
+}
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java
new file mode 100644
index 000000000..8427b5aa7
--- /dev/null
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java
@@ -0,0 +1,135 @@
+package com.fangjia.dal.mongo.open.model;
+
+import com.fangjia.dal.MongoEntity;
+
+import javax.persistence.*;
+import java.util.Date;
+
+/**
+ * 小区周边配套统计信息
+ *
+ * @author shenjian@fangjia.com
+ */
+@Entity
+@Table(name = "fangjia_stat.community_poi_gj")
+public class CommunityPoi implements MongoEntity {
+
+ private static final long serialVersionUID = -7663644539542318409L;
+
+ private String id;
+ private String city;
+ private String region;
+ private String name;
+ private Integer subway;
+ private Integer traffic;
+ private Integer bank;
+ private Integer primary;
+ private Integer junior;
+ private Integer hospital;
+ /**
+ * 不利设施,丧葬
+ */
+ private Integer funeral;
+
+ @Id
+ @Column(name = "_id")
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ @Override
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ @Column(name = "city")
+ public String getCity() {
+ return city;
+ }
+
+ public void setCity(String city) {
+ this.city = city;
+ }
+
+ @Column(name = "region")
+ public String getRegion() {
+ return region;
+ }
+
+ public void setRegion(String region) {
+ this.region = region;
+ }
+
+ @Column(name = "name")
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ @Column(name = "subway")
+ public Integer getSubway() {
+ return subway;
+ }
+
+ public void setSubway(Integer subway) {
+ this.subway = subway;
+ }
+
+ @Column(name = "traffic")
+ public Integer getTraffic() {
+ return traffic;
+ }
+
+ public void setTraffic(Integer traffic) {
+ this.traffic = traffic;
+ }
+
+ @Column(name = "bank")
+ public Integer getBank() {
+ return bank;
+ }
+
+ public void setBank(Integer bank) {
+ this.bank = bank;
+ }
+
+ @Column(name = "primary")
+ public Integer getPrimary() {
+ return primary;
+ }
+
+ public void setPrimary(Integer primary) {
+ this.primary = primary;
+ }
+
+ @Column(name = "junior")
+ public Integer getJunior() {
+ return junior;
+ }
+
+ public void setJunior(Integer junior) {
+ this.junior = junior;
+ }
+
+ @Column(name = "hospital")
+ public Integer getHospital() {
+ return hospital;
+ }
+
+ public void setHospital(Integer hospital) {
+ this.hospital = hospital;
+ }
+
+ @Column(name = "funeral")
+ public Integer getFuneral() {
+ return funeral;
+ }
+
+ public void setFuneral(Integer funeral) {
+ this.funeral = funeral;
+ }
+}
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java
index bf43fa450..69108673a 100644
--- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java
@@ -67,12 +67,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i
must.add(addTerms("district_name", alias));
must.add(addMatch("cat", "sell"));
must.add(addRange("avg_price", minPrice, maxPrice));
-
- JSONObject term = new JSONObject();
- JSONObject match = new JSONObject();
- match.put("p_date", 20210501);
- term.put("match", match);
- must.add(term);
+ must.add(addRangeGte("p_date", 20210501));
bool.put("must", must);
query.put("bool", bool);
@@ -168,6 +163,16 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i
return term;
}
+ private JSONObject addRangeGte(String name, Number min){
+ JSONObject term = new JSONObject();
+ JSONObject range = new JSONObject();
+ JSONObject rangeValue = new JSONObject();
+ rangeValue.put("gte", min);
+ range.put(name, rangeValue);
+ term.put("range", range);
+ return term;
+ }
+
private List wrapSeawater(List jsonList){
List seawaterList= new ArrayList();
@@ -210,6 +215,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i
seawater.setFloor(source.getInt("floor"));
}catch(Exception e){
seawater.setFloor(0);
+ }
+ try{
+ seawater.setFloorInfo(source.getString("floor_info"));
+ }catch(Exception e){
+
}
if(source.containsKey("fitment")){
seawater.setFitment(source.getString("fitment"));
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java
index 910d805ea..1af563193 100644
--- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java
+++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java
@@ -48,6 +48,7 @@ public class Seawater implements SeawaterEntity {
private Double area;
private Integer height;
private Integer floor;
+ private String floorInfo;
private String fitment;
private Integer room;
private Integer hall;
@@ -198,6 +199,15 @@ public class Seawater implements SeawaterEntity {
this.floor = floor;
}
+ @Column(name = "floorInfo")
+ public String getFloorInfo() {
+ return floorInfo;
+ }
+
+ public void setFloorInfo(String floorInfo) {
+ this.floorInfo = floorInfo;
+ }
+
@Column(name = "fitment")
public String getFitment() {
return fitment;
diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java
index 0b0dce39f..1874f3500 100644
--- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java
+++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java
@@ -31,12 +31,15 @@ import javax.validation.Valid;
import com.fangjia.core.util.*;
import com.fangjia.dal.*;
import com.fangjia.dal.mongo.model.*;
+import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.open.dto.*;
import com.fangjia.service.district.*;
import com.fangjia.service.mongo.fangjia.*;
import com.fangjia.service.open.*;
+import com.fangjia.service.open.impl.CommunityPoiServiceImpl;
import com.fangjia.service.util.DistrictScoreUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
@@ -175,6 +178,9 @@ public class PropertyController {
private SeawaterDao seawaterDao;
private SeawaterService seawaterService;
private SqmetreRentService sqmetreRentService;
+ private CommunityPoiService communityPoiService;
+ private CommunityBasePriceService communityBasePriceService;
+ private EvaluateFactorService evaluateFactorService;
private FileDao fileDao = FileDao.getInstance();
private static final int DEFAULT_PAGE_SIZE = 20;
@@ -390,10 +396,31 @@ public class PropertyController {
put("5000", "five");// 五线
}
};
+ /**
+ * 第三方均价映射
+ */
+ private static final Map OTHER_PRICE_MAP = new HashMap() {
+ {
+ put("zhuge_price", "诸葛找房");
+ put("anjuke_price", "安居客");
+ put("fang_price", "房天下");
+ put("lianjia_price", "链家");
+ put("cityhouse_price", "禧泰数据");
+ put("maitian_price", "麦田");
+ put("ke_price", "贝壳");
+ put("taofang_price", "淘房365");
+ put("zhongyuan_price", "中原");
+ put("tbs_price", "兔博士");
+ put("qfang_price", "Q房");
+ put("fangtan_price", "房探");
+ put("fangduoduo_price", "房多多");
+ put("wiwj_price", "我爱我家");
+ }
+ };
/**
* 利用百度API把GPS转换成Google经纬度
- *
+ *
* @param lng
* @param lat
* @return
@@ -415,7 +442,7 @@ public class PropertyController {
/**
* 利用算法模板把GPS转换成Google经纬度
- *
+ *
* @param lng
* @param lat
* @return
@@ -607,7 +634,7 @@ public class PropertyController {
/**
* 判断小区是否有经纬度
- *
+ *
* @param district
* @return
*/
@@ -618,7 +645,7 @@ public class PropertyController {
/**
* 调用匹配接口匹配小区
- *
+ *
* @param city
* @param address
* @param region
@@ -992,6 +1019,32 @@ public class PropertyController {
}
}
+ // 调整面积不同面积价格
+ try {
+ DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
+ int orgAvgPrice = 0;
+ if (districtStatBo != null) {
+ if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
+ orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
+ } else if (districtStatBo.getDistrictStat() != null) {
+ orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
+ }
+ }
+ if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
+ int factorPrice = evaluation.getAvgPrice();
+ int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
+ Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
+ Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
+ // 偏离度超过5%不进行调整
+ if (minPrice <= avgPrice && avgPrice <= maxPrice) {
+ evaluation.setAvgPrice(avgPrice);
+ evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
+ }
+ }
+ } catch (Exception ex) {
+
+ }
+
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
Map districtPriceMap = new HashMap();
@@ -1162,19 +1215,52 @@ public class PropertyController {
evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
return mav;
+ }
+ // 调整面积不同面积价格
+ try {
+ DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
+ int orgAvgPrice = 0;
+ if (districtStatBo != null) {
+ if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
+ orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
+ } else if (districtStatBo.getDistrictStat() != null) {
+ orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
+ }
+ }
+ if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
+ int factorPrice = evaluation.getAvgPrice();
+ int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
+ Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
+ Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
+ // 偏离度超过5%不进行调整
+ if (minPrice <= avgPrice && avgPrice <= maxPrice) {
+ evaluation.setAvgPrice(avgPrice);
+ evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
+ }
+ }
+ } catch (Exception ex) {
+
}
Map districtPriceMap = new LinkedHashMap();
districtPriceMap.put("city", district.getCity());
districtPriceMap.put("district", district.getRegion());
districtPriceMap.put("name", district.getName());
long totalPrice = evaluation.getBigTotalPrice();
- if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
- totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
- evaluation.setBigTotalPrice(totalPrice);
- evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
- }
+ // 取消系数下调
+// if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
+// totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
+// evaluation.setBigTotalPrice(totalPrice);
+// evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
+// }
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
+ // 新增置信度
+ CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId());
+ if (communityBasePrice != null) {
+ districtPriceMap.put("credit", communityBasePrice.getCredit());
+ } else {
+ districtPriceMap.put("credit", evaluation.getCredit());
+ }
List names = new ArrayList();
names.add(district.getName());
if(district.getAlias() != null){
@@ -1186,6 +1272,31 @@ public class PropertyController {
}
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
+ Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9);
+ Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1);
+ // 新增第三方均价
+ try {
+ double dCredit = 0.61;
+ Map rMap = getLastEvaluationPrice(district, evaluation, dCredit);
+ if (null != rMap) {
+ if (rMap.containsKey("other_price")) {
+ Map otherPriceMap = (Map) rMap.get("other_price");
+ Map priceResultMap = new HashMap();
+ for (String key : otherPriceMap.keySet()) {
+ Integer price = otherPriceMap.get(key);
+ // 剔除偏差大于10的
+ if (min10Price <= price && max10Price >= price) {
+ if (OTHER_PRICE_MAP.containsKey(key)) {
+ priceResultMap.put(OTHER_PRICE_MAP.get(key), price);
+ }
+ }
+ }
+ districtPriceMap.put("otherPrice", priceResultMap);
+ }
+ }
+ } catch (Exception ex) {
+
+ }
List houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
if(houses.size() > 3){
districtPriceMap.put("calcType", 1);
@@ -1214,7 +1325,7 @@ public class PropertyController {
}else if(district.getLng() != null && district.getLng() > 0){
double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6);
double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6);
- List districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 4, null);
+ List districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 6, null);
if(districts.size() < 2){
evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
@@ -1252,9 +1363,15 @@ public class PropertyController {
districtDto.append("buildYear", String.valueOf(buildYear));
}
districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d));
- if(districtAnalyData != null && districtAnalyData.getHalfYearMarkup() != null){
- districtDto.append("halfYearMarkup",DataUtil.roundDouble( districtAnalyData.getHalfYearMarkup() * 100 ,2));
+ if(districtAnalyData != null){
+ if (districtAnalyData.getHalfYearMarkup() != null) {
+ districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2));
+ }
+ if (districtAnalyData.getAvgPrice() != null && minPrice <= districtAnalyData.getAvgPrice() && maxPrice >= districtAnalyData.getAvgPrice()) {
+ districtDto.append("avgPrice", districtAnalyData.getAvgPrice());
+ }
}
+
nearCommunityDtos.add(districtDto);
}
Integer buildYear = districtService.getBuildYear(district);
@@ -1262,6 +1379,18 @@ public class PropertyController {
modelFeaturesFto.put("buildYear", String.valueOf(buildYear));
}
modelFeaturesFto.append("nearCommunity", nearCommunityDtos);
+
+ // 模型特征新增周边数量
+ CommunityPoi communityPoi = communityPoiService.getById(district.getId());
+ if (communityPoi != null) {
+ modelFeaturesFto.put("subway", communityPoi.getSubway());
+ modelFeaturesFto.put("traffic", communityPoi.getTraffic());
+ modelFeaturesFto.put("bank", communityPoi.getBank());
+ modelFeaturesFto.put("primary", communityPoi.getPrimary());
+ modelFeaturesFto.put("hospital", communityPoi.getHospital());
+ modelFeaturesFto.put("junior", communityPoi.getJunior());
+ modelFeaturesFto.put("funeral", communityPoi.getFuneral());
+ }
districtPriceMap.put("modelFeatures", modelFeaturesFto);
}
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
@@ -1469,7 +1598,7 @@ public class PropertyController {
return mav;
}
}
-
+
/**
* 四川锦程 定制接口 多建筑年代
* @param model
@@ -1581,7 +1710,7 @@ public class PropertyController {
districtPriceMap.put("buildYear", matcher3.group());
}
}
-
+
// 根据城市等 级设置D级可信度
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
@@ -2094,7 +2223,7 @@ public class PropertyController {
/**
* 小区租金查询接口
- *
+ *
* @param model
* @return
*/
@@ -2573,7 +2702,7 @@ public class PropertyController {
/**
* 批量获取小区的均价,均价走势数据
- *
+ *
* @param request
* @return
*/
@@ -2715,7 +2844,7 @@ public class PropertyController {
/**
* 批量获取小区的配套数据,最多12个小区
- *
+ *
* @param request
* @return
*/
@@ -2860,7 +2989,7 @@ public class PropertyController {
/**
* 获取JSON参数中的num参数值
- *
+ *
* @param jo
* @return
*/
@@ -2885,7 +3014,7 @@ public class PropertyController {
/**
* 获取JSON参数中的radius值
- *
+ *
* @param jo
* @return
*/
@@ -3693,7 +3822,7 @@ public class PropertyController {
/**
* 解析出关键字中的小区名
- *
+ *
* @param keyword
* @return
*/
@@ -3810,7 +3939,7 @@ public class PropertyController {
/**
* 小区统计分析接口
- *
+ *
* @param model
* @return
*/
@@ -3895,7 +4024,7 @@ public class PropertyController {
/**
* 小区交通接口
- *
+ *
* @param model
* @return
*/
@@ -3953,7 +4082,7 @@ public class PropertyController {
/**
* 小区周边接口
- *
+ *
* @param model
* @return
*/
@@ -4012,7 +4141,7 @@ public class PropertyController {
/**
* 相同小区相似户型房源挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
- *
+ *
* @author yinjihuan
* @date 2016年9月1日
* @param city
@@ -4052,7 +4181,7 @@ public class PropertyController {
/**
* 周边小区相似户型挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
- *
+ *
* @author yinjihuan
* @date 2016年9月1日
* @param city
@@ -4103,7 +4232,7 @@ public class PropertyController {
/**
* 小区周边一公里均价
- *
+ *
* @author wenyan
* @param model
* @return
@@ -4126,7 +4255,7 @@ public class PropertyController {
/**
* 相同小区相似户型租金范围查询接口
- *
+ *
* @param model
* @return
*/
@@ -4169,7 +4298,7 @@ public class PropertyController {
/**
* 周边小区相似户型租金范围查询接口
- *
+ *
* @param model
* @return
*/
@@ -4232,7 +4361,7 @@ public class PropertyController {
/**
* 室号查询接口
- *
+ *
* @param model
* @return
*/
@@ -4268,7 +4397,7 @@ public class PropertyController {
/**
* 楼栋楼层信息查询接口
- *
+ *
* @param model
* @return
*/
@@ -4293,7 +4422,7 @@ public class PropertyController {
/**
* 楼栋查询接口
- *
+ *
* @param model
* @return
*/
@@ -4338,7 +4467,7 @@ public class PropertyController {
/**
* 楼栋查询接口
- *
+ *
* @param model
* @return
*/
@@ -4791,7 +4920,7 @@ public class PropertyController {
/**
* 房产交易记录查询
- *
+ *
* @param model
* @return
*/
@@ -4851,7 +4980,7 @@ public class PropertyController {
/**
* 房产拍卖记录
- *
+ *
* @param model
* @return
*/
@@ -4912,7 +5041,7 @@ public class PropertyController {
/**
* 房产近期评估次数
- *
+ *
* @param model
* @return
*/
@@ -5627,4 +5756,16 @@ public class PropertyController {
public void setRentDealPriceService(RentDealPriceService rentDealPriceService) {
this.rentDealPriceService = rentDealPriceService;
}
+
+ public void setCommunityPoiService(CommunityPoiService communityPoiService) {
+ this.communityPoiService = communityPoiService;
+ }
+
+ public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) {
+ this.evaluateFactorService = evaluateFactorService;
+ }
+
+ public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
+ this.communityBasePriceService = communityBasePriceService;
+ }
}
diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java b/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java
index e16664d16..37805ef21 100644
--- a/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java
+++ b/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java
@@ -75,8 +75,8 @@ public class AnalyticsFilter implements Filter {
if ("1".equals(request.getParameter("__refresh_cache"))) {
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
}
- createUID(request, response);
- createSID(request, response);
+ //createUID(request, response);
+ //createSID(request, response);
response.addHeader("Access-Control-Allow-Origin", "*");
response.addHeader("Access-Control-Allow-Methods", "*");
chain.doFilter(request, response);
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy
index c2f8fe431..6cd17549e 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
- public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{
}else{
def jsonObj = null
if(dist.address){
- jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
+// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
+ jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
}
- if(jsonObj && jsonObj.searchAddress){
- ret << jsonObj.searchAddress.street
- ret << jsonObj.searchAddress.streetNumber
+ if(jsonObj && jsonObj.street){
+ if(jsonObj.street){
+ ret << jsonObj.street
+ }else{
+ ret << ""
+ }
+
+ if(jsonObj.streetNumber){
+ ret << jsonObj.streetNumber
+ }else{
+ ret << ""
+ }
+
}else{
ret << ""
ret << ""
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy
index 4978bd962..0bc461cc5 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
- exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419);
+ exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
- def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
+ def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
@@ -408,7 +408,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
- def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
+ def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -449,6 +449,13 @@ class BigdataForsaleBatch extends BaseBatch{
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
+ if(estate_type2 && estate_type2=="普通住宅"){
+ if(seawater.estate_type&& seawater.estate_type.contains("商住")){
+ println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
+ continue;
+ }
+ }
+
if(!seawater.total_price){
continue;
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy
index 3fffa870c..74474aff6 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy
@@ -57,6 +57,12 @@ class CcbFtpUtils {
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
+ static def listFileNames(month){
+ //上传到ftp
+ sftpChannel = reConnect();
+ Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
+ return files
+ }
static main(args) {
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
index d7840e5bb..c8697c956 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy
@@ -15,8 +15,8 @@ class DealPriceDataProcess {
- def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
- "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
+// def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
+// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -373,16 +373,17 @@ class DealPriceDataProcess {
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
- def month = 202203
+ def month = 202204
def rows = []
for(def city : citys) {
- def rets = findPeGteThirdParty(city, 202204)
+ def rets = findPeGteThirdParty(city, 202205)
def total = 0
+ println city +" "+rets.size()
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql)
}
- println city +" "+rets.size()
+
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy
index 060914122..af8723b42 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy
@@ -38,8 +38,8 @@ class ExportOracleData {
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
- exportBigdataForsale("2022-03-19")
- exportBigdataDeal("2022-03-01")
+ exportBigdataForsale("2022-04-19")
+ exportBigdataDeal("2022-04-01")
//计算结果相关
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy
index edc128cf1..7258934fa 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy
@@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
+import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
+import org.apache.commons.codec.digest.DigestUtils
class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
@@ -33,6 +35,8 @@ class ExportOracleEvaluationData {
exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
+// 生成MD5文件
+ exportMd5()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// ok文件
@@ -413,4 +417,23 @@ class ExportOracleEvaluationData {
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
}
+ def exportMd5(){
+ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
+ Vector files = CcbFtpUtils.listFileNames(curDate)
+ Iterator fileVec = files.iterator()
+ def fileRows = []
+ while(fileVec.hasNext()){
+ ChannelSftp.LsEntry file = fileVec.next()
+ if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
+ fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
+ }
+ }
+ def fileName = "/ccb/md5.txt"
+ if(CommonDatautils.isWindow()){
+ fileName = "D:\\房价网\\md5.txt"
+ }
+ FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
+ CcbFtpUtils.ftpUpload(null, fileName, curDate)
+ println "md5.txt 上传Ftp完成......"
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy
index 42ad78607..b0476927d 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy
@@ -1,21 +1,29 @@
package com.fangjia.script.ccbevaluation
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
+import groovy.sql.Sql
+import org.bson.types.ObjectId
class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
+ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
+ "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
+
static main(args) {
def task = new LoadGuide()
task.run()
}
def run(){
+ addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
@@ -26,9 +34,51 @@ class LoadGuide {
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
- savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
+// savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
}
+
+ def addPersonGuided(filename){
+ ExcelUtils.modify(filename, { d ->
+ def city = d['B']
+ if(city){
+ def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
+
+ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
+ createSql( d, cityCodeDatas, distCodeMap)
+ }
+
+ })
+ }
+
+ def createSql(d, cityCodeDatas, distCodeMap){
+ def id = new ObjectId()
+ def commid = d['A']
+ def cityId = cityCodeDatas.get(d['B'])
+ def sampleRegion = d['C'].replace("街道","").replace("镇","")
+ def distRegion = distCodeMap.get(sampleRegion)
+ def cityName = d['B']
+ def distName = d['C']
+ def commName = d['D']
+ def price = d['E'].toDouble().toInteger()
+
+ def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district")
+ if(dist){
+ def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
+ def dealDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
+
+
+ def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " +
+ "('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')"
+ println sql
+ sqlInstance.executeUpdate(sql)
+ }else{
+ println " 没查到 "+commName
+ }
+
+
+ }
+
def match(def filename){
def city = "西安"
ExcelUtils.modify(filename,{d->
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy
new file mode 100644
index 000000000..93f18dfce
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrict2Mongo85.groovy
@@ -0,0 +1,67 @@
+package com.fangjia.script.ccbevaluation
+
+import com.fangjia.core.RegionData
+import com.fangjia.core.util.DistrictCodeUtils
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.gj.district.FangjiaClientUtils
+import com.fangjia.script.lixue.util.DBUtils
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.ElasticUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.mongodb.DBObject
+import groovy.sql.Sql
+import org.apache.lucene.queryparser.classic.QueryParser
+import org.apache.poi.ss.util.RegionUtil
+import org.bson.types.ObjectId
+import org.elasticsearch.index.query.BoolQueryBuilder
+import org.elasticsearch.index.query.QueryBuilder
+import org.elasticsearch.index.query.QueryBuilders
+import org.elasticsearch.xpack.sql.jdbc.EsDataSource
+
+import java.sql.ResultSet
+import java.util.concurrent.ConcurrentHashMap
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+
+/**
+ * oracle中小区统计数据同步至85中
+ */
+class OracleDistrict2Mongo85 {
+
+ static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
+ "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
+
+ static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017)
+
+ static main(args) {
+ // 只写入重庆市数据
+ def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ"
+ def count = 0
+ sqlInstance.eachRow(sql, { r ->
+ def city = DistrictCodeUtils.getCityName(r['cityid'])
+ def region = RegionData.formatRegion(city, r['distname'])
+ DBObject dbObject = new BasicDBObject("_id", r['commid'])
+ DBObject updateObject = new BasicDBObject("city", city).append("region", region)
+ .append("name", r['commname'])
+ .append('subway', Integer.parseInt(r['subway'].toString()))
+ .append("traffic", Integer.parseInt(r['traffic'].toString()))
+ .append("bank", Integer.parseInt(r['bank'].toString()))
+ .append("primary", Integer.parseInt(r['primary'].toString()))
+ .append("junior", Integer.parseInt(r['junior'].toString()))
+ .append("hospital", Integer.parseInt(r['hospital'].toString()))
+ .append("funeral", Integer.parseInt(r['funeral'].toString()))
+ .append("c_date", new Date());
+ communityPoiConn.update(dbObject, updateObject, true, false)
+ count++;
+ if (count % 500 == 0) {
+ println "已写入" + count + "条小区数据"
+ }
+ });
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy
new file mode 100644
index 000000000..a824a3d7e
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/OracleDistrictBasePrice2Mongo85.groovy
@@ -0,0 +1,52 @@
+package com.fangjia.script.ccbevaluation
+
+import com.fangjia.core.RegionData
+import com.fangjia.core.util.DistrictCodeUtils
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.gj.district.FangjiaClientUtils
+import com.fangjia.script.lixue.util.DBUtils
+import com.fangjia.script.util.DataUtil
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.ElasticUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.mongodb.DBObject
+import groovy.sql.Sql
+
+/**
+ * oracle中小区置信度数据同步至85中
+ */
+class OracleDistrictBasePrice2Mongo85 {
+
+ static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
+ "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
+
+ static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017)
+
+ static main(args) {
+ def sql = "SELECT cityname, distname, commid, commname, confidence FROM community_base_price_gj WHERE YEAR_MONTH='202205'"
+ def count = 0
+ sqlInstance.eachRow(sql, { r ->
+ def city = r['cityname']
+ def region = RegionData.formatRegion(city, r['distname'])
+ DBObject dbObject = new BasicDBObject("_id", r['commid'])
+ if (r['confidence'] != null) {
+ def credit = new BigDecimal(r['confidence']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble()
+ DBObject updateObject = new BasicDBObject("city", city).append("region", region)
+ .append("name", r['commname'])
+ .append("credit", credit)
+ .append("c_date", new Date());
+ communityBasePriceConn.update(dbObject, updateObject, true, false)
+ count++;
+ } else {
+ print "置信度为空:" + r.toString()
+ }
+ if (count % 500 == 0) {
+ println "已写入" + count + "条小区数据"
+ }
+ });
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy
index e262bd7c5..d1cc4209b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
- def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
- def startDate = 202201
- def endDate = 202203
+ def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
+ def startDate = 202202
+ def endDate = 202204
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy
index 3ed13ebfc..3b3e6a7c5 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
-// task.run()//统计最近3个月抓取的成交数据量
- task.exportTrade()//导出未入库成交
+ task.run()//统计最近3个月抓取的成交数据量
+// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
- rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
+ rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
- def startDate = "2022-01-01"
- def date = "2022-02-01"
- def endDate = "2022-03-01"
- def newDate = "2022-04-01"
+ def startDate = "2022-02-01"
+ def date = "2022-03-01"
+ def endDate = "2022-04-01"
+ def newDate = "2022-05-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
}
- ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据")
+ ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy
new file mode 100644
index 000000000..1aa5a391f
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy
@@ -0,0 +1,155 @@
+package com.fangjia.script.ccbevaluation.data
+
+import com.fangjia.script.ccbevaluation.CcbCommonUtils
+import com.fangjia.script.common.DataHandleUtils
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import groovy.sql.Sql
+
+class CcbDataStat {
+
+ public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
+
+ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
+ "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
+
+ static main(args) {
+ def task = new CcbDataStat()
+// task.run()
+ task.tradeRun()
+ }
+
+ def tradeRun(){
+ def ccbCitys = CcbCommonUtils.citys;
+ def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")
+ def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d ->
+ if(d['A']){
+ if(ccbCitys.contains(d['A'])){
+ d['D'] = findOracleTotalCount(d['A'])
+ }else{
+ def totalcount = findAllTradeCount(d['A'], startDate, endDate)
+ def repeatCount = findRepeatCount(d['A'], startDate, endDate)
+ def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate)
+ def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate)
+ d['B'] = totalcount
+ d['C'] = repeatCount
+ d['D'] = matchTotalCount
+ d['E'] = matchRepeatTotalCount
+ }
+
+ }
+ })
+ }
+
+
+
+ def findRepeatCount(city, startDate, endDate){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
+
+ def dbcursor = DEAL_PRICE_COLL.find(q)
+ def set = new HashSet()
+ for(def d : dbcursor){
+ def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
+ if(!set.contains(key)){
+ set << key
+ }
+ }
+ return set.size()
+ }
+
+ def findMatchRepeatTradeCount(city, startDate, endDate){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
+ q.put("match_id", new BasicDBObject('$exists', true))
+ def dbcursor = DEAL_PRICE_COLL.find(q)
+ def set = new HashSet()
+ for(def d : dbcursor){
+ def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
+ if(!set.contains(key)){
+ set << key
+ }
+ }
+ return set.size()
+ }
+
+
+ def findMatchTradeCount(city, startDate, endDate){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
+ q.put("match_id", new BasicDBObject('$exists', true))
+ def count = DEAL_PRICE_COLL.count(q)
+ return count
+ }
+
+
+ public static def findAllTradeCount(city, startDate, endDate){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
+// q.put("match_id", new BasicDBObject('$exists', false))
+ def total = DEAL_PRICE_COLL.count(q)
+ return total
+ }
+
+ def findOracleTotalCount(city){
+ def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate
+ count = r.count
+ })
+ return count
+ }
+
+ def run(){
+ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
+ if(d['A']) {
+ def citycode = cityCodeDataMap.get(d['A'])
+ def total = findTotalCount(citycode)
+ def f_count = findDistFinishDate(citycode)
+ def b_count = findBuildTypeCount(citycode)
+ def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
+ def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
+ d['C'] = f_pre
+ d['D'] = b_pre
+ d['E'] = total
+ println d['A'] +" "+ total +" "+f_count+" "+b_count
+ }
+ })
+ }
+
+ def findTotalCount(citycode){
+ def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100)
+ def count = 0
+ sqlInstance.eachRow(sql, { r ->
+ count = r.count
+ })
+ return count
+ }
+
+ def findDistFinishDate(citycode){
+ def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100)
+ def count = 0
+ sqlInstance.eachRow(sql, { r ->
+ count = r.count
+ })
+ return count
+ }
+
+ def findBuildTypeCount(citycode){
+ def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100)
+ def count = 0
+ sqlInstance.eachRow(sql, { r ->
+ count = r.count
+ })
+ return count
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy
new file mode 100644
index 000000000..32725d092
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistNameMatch.groovy
@@ -0,0 +1,117 @@
+package com.fangjia.script.ccbevaluation.data
+
+import com.fangjia.core.RegionData
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.OpenApiClient
+import groovy.sql.Sql
+
+class CcbDistNameMatch {
+
+ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
+ "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
+
+ static main(args) {
+ def task = new CcbDistNameMatch();
+// task.run()
+ task.run1()
+ }
+
+ def run1(){
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
+ if (d['A']) {
+ def region = d['B']
+ def newRegion = RegionData.formatRegion(d['A'], region)
+ if (newRegion) {
+ if(d['D'].contains("、")){
+ println d['D']
+ }
+ def arrs = d['D'].split("、")
+ def list = []
+ def flags = []
+ for(def s : arrs){
+ def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district")
+ if(dist){
+ list << dist.name
+ flags << "完全一致"
+ }else{
+ def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion)
+ if(jsonObject != null && jsonObject.precise!=-1) {
+ if(jsonObject.get("district") == newRegion){
+ list << jsonObject.get("name")
+ flags << "匹配接口"
+ }
+ }
+ }
+
+ }
+ if(list){
+ list = list.unique()
+ d['E'] = list.join(",")
+ d['F'] = flags.join(",")
+ }
+ }
+ }
+ })
+ }
+
+ def run(){
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d ->
+ def regionCode = null
+ if (d['A']) {
+ def region = d['B']
+ def newRegion = RegionData.formatRegion(d['A'], region)
+ if (newRegion) {
+ def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
+ if(dist){
+ d['D'] = dist.name
+ def count = findDistrictByOracle(dist._id.toString())
+ if(count && count>0){
+ d['F'] = 1
+ }else{
+ d['H'] = dist.estate_type2
+ d['J'] = dist.name_cat
+ }
+ }else{
+ def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
+ if (rets) {
+ if(rets.size()>1){
+ d['E'] = "别名-多个"
+ }else{
+ dist = rets[0]
+ d['E'] = d['C'] +"--"+dist.name
+ def count = findAliasByOracle(dist._id.toString())
+ if(count && count>0) {
+ d['G'] = 1
+ }else{
+ d['H'] = dist.estate_type2
+ d['J'] = dist.name_cat
+ }
+ }
+ }
+ }
+ }
+ }
+ });
+ }
+
+ def findDistrictByOracle(commid){
+ def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
+ def count = 0
+
+ sqlInstance.eachRow(sql,{r->
+ count = r.count
+ });
+ return count
+ }
+
+ def findAliasByOracle(commid){
+ def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'";
+ def count = 0
+
+ sqlInstance.eachRow(sql,{r->
+ count = r.count
+ });
+ return count
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy
index 2db54e10a..98d755434 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy
@@ -7,6 +7,7 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
+import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
@@ -26,9 +27,9 @@ class CcbDistrictToDB {
// task.run()
// task.run1()
// task.meituanRun()
-// task.meituanSqrentRun()
+ task.meituanSqrentRun()
// task.updateDistrict()
- task.updateDongguangRegion()
+// task.updateDongguangRegion()
}
def updateDongguangRegion(){
@@ -80,13 +81,17 @@ class CcbDistrictToDB {
}
def meituanSqrentRun(){
- ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
+ def index = 0
+ ExcelUtils.modify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
+ index++
+ println index+" "+d['H']
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion) {
- def name = d['H']
+ def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name)
+
if(name){
def estateType2 = "普通住宅"
if(name.contains("别墅")){
@@ -103,7 +108,6 @@ class CcbDistrictToDB {
}
}
}
-
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
}
@@ -115,97 +119,100 @@ class CcbDistrictToDB {
def index = 0//美团需入库小区合并后.xlsx
def keys = new CopyOnWriteArrayList()
- ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d ->
+ ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
+ index++
+ println index+" "+d['H']
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
- if(newRegion){
- def name = d['H']
+ if(newRegion) {
+ def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name)
+ if (name) {
def formatAddress = null
- if(d['E']){
- formatAddress = AddressUtil.formatAddress(city, d['E'])
+ if (d['I']) {
+ formatAddress = AddressUtil.formatAddress(city, d['I'])
}
- if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
- def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
- def map = findStandardObject2(city,region, name, d['E'], rets);
- if(map && map.address){
- formatAddress = AddressUtil.formatAddress(city, map.address)
- if(!formatAddress){
- if(map.address.contains("交叉口")){
- formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
- }else if(map.address.contains("交叉")){
- formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
- }
- }
- }
- }
+// if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
+// def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
+// def map = findStandardObject2(city,region, name, d['E'], rets);
+// if(map && map.address){
+// formatAddress = AddressUtil.formatAddress(city, map.address)
+// if(!formatAddress){
+// if(map.address.contains("交叉口")){
+// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
+// }else if(map.address.contains("交叉")){
+// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
+// }
+// }
+// }
+// }
Long glat = null
Long glng = null
Long blat = null
Long blng = null
- if(d['K']){
- glat = d['L'].toString().toDouble() * 10000000000L
- glng = d['K'].toString().toDouble() * 10000000000L
- def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
+ if (d['L']) {
+ glat = d['M'].toString().toDouble() * 10000000000L
+ glng = d['L'].toString().toDouble() * 10000000000L
+ def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
- if(name.contains("别墅")){
+ if (name.contains("别墅")) {
estateType2 = "别墅"
- }else if(name.contains("商住")){
+ } else if (name.contains("商住")) {
estateType2 = "商住"
- }else{
- def type = d['I']
- if(type){
- if(type.contains("别墅")){
+ } else {
+ def type = d['J']
+ if (type) {
+ if (type.contains("别墅")) {
estateType2 = "别墅"
- }else if(type.contains("商住")){
+ } else if (type.contains("商住")) {
estateType2 = "商住"
}
}
}
- def key = city+"##"+newRegion+"##"+name
- if(name && !keys.contains(key)){
+ def key = city + "##" + newRegion + "##" + name
+ if (name && !keys.contains(key)) {
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
- if(!dist){
+ if (!dist) {
keys << key
def data = new BasicDBObject();
- data.put("name_cat",0)
- data.put("status",0)
- data.put("cat","district")
+ data.put("name_cat", 12)
+ data.put("status", 0)
+ data.put("cat", "district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
- data.put("region",newRegion)
+ data.put("region", newRegion)
data.put("name", name)
- data.put("estate_type2",estateType2)
+ data.put("estate_type2", estateType2)
data.put("address", formatAddress)
- if(glat){
- data.put("lat",glat)
- data.put("lng",glng)
- data.put("lat2",blat)
- data.put("lng2",blng)
+ if (glat) {
+ data.put("lat", glat)
+ data.put("lng", glng)
+ data.put("lat2", blat)
+ data.put("lng2", blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
- data.put("mark_map",markMap)
+ data.put("mark_map", markMap)
index++
DbUtils.seaweed.save(data);
- println index+" "+data
+ println index + " " + data
def stop = false
- while (!stop){
- if(data._id.toString()){
+ while (!stop) {
+ if (data._id.toString()) {
addPrice(data)
stop = true;
}
@@ -213,19 +220,17 @@ class CcbDistrictToDB {
}
}
+ }
}else{
println city +" "+region +" 区域无法标准化"
}
},0);
-
-
-
}
def addPrice(dist){
- def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
- def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
+ def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01")
+ def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
@@ -435,9 +440,12 @@ class CcbDistrictToDB {
}
def nameFileter(def name){
- name = name.replace("·","").replace(".","").replace("|","")
- name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
- name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
+ if(name){
+ name = name.replace("·","").replace(".","").replace("|","")
+ name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
+ name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
+ }
+
return name;
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy
index a1e095a7f..afa2233b6 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy
@@ -1,10 +1,12 @@
package com.fangjia.script.ccbevaluation.data
+import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
+import com.mongodb.DBObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import com.fangjia.core.util.BaiDuApiUtils
@@ -13,7 +15,82 @@ class DistrictMarge {
static main(args) {
def task = new DistrictMarge()
//task.run()
- task.repeatAddress()
+// task.repeatAddress()
+// task.removeDist()
+ task.updateAddress()
+// task.addMappingAlias()
+ }
+ def updateAddress(){
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d ->
+ if (d['A']) {
+ def region = d['B'].trim()
+ def newRegion = RegionData.formatRegion(d['A'], region)
+ if (newRegion) {
+ def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
+ if (dist) {
+ DBObject udata = new BasicDBObject()
+ udata.put("address", d['D'])
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
+ }
+ }else{
+ println d['A'] + " 格式化失败 "+d['B']
+ }
+ }
+ });
+ }
+
+ def removeDist(){
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d ->
+ if (d['A']) {
+ def region = d['B']
+ def newRegion = RegionData.formatRegion(d['A'], region)
+ if (newRegion) {
+ def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
+ if (dist) {
+ DBObject udata = new BasicDBObject()
+ udata.put("status", 1)
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
+ }
+ }else{
+ println d['A'] + " 格式化失败 "+d['B']
+ }
+ }
+ })
+ }
+
+ def addMappingAlias(){
+ def index = 0
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
+ index ++
+ if (d['A']) {
+ def region = d['B'].trim()
+ def newRegion = RegionData.formatRegion(d['A'], region)
+ if (newRegion) {
+ def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
+ if (dist) {
+ def mapping_alias = dist.mapping_alias
+ if(!dist.mapping_alias){
+ mapping_alias = []
+ }
+ def alias = d['D']
+ def arrs = alias.split("、")
+ for(def s : arrs){
+ if(!mapping_alias.contains(s)){
+ mapping_alias << s
+ }
+ }
+ DBObject udata = new BasicDBObject()
+ udata.put("mapping_alias", mapping_alias)
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
+ }
+ }else{
+ println d['A'] + " 格式化失败 "+d['B']
+ }
+ }
+ })
}
def repeatAddress(){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy
index 2f5d02c2f..a62cf530f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy
@@ -36,10 +36,37 @@ class MatchMapData {
// task.run3()
// task.exportHouse()
// task.addDistrictLevel()
- task.run4()
+// task.run4()
+ task.run5()
}
+ def run5(){
+ ExcelUtils.multiThreadingModify("D:\\数据\\建行\\数据核实.xlsx", { d ->
+ if (d['A']) {
+ def region = d['B']
+ def newRegion = RegionData.formatRegion(d['A'], region)
+ if (newRegion) {
+ def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
+ if(dist){
+ if(dist.mapping_alias){
+ d['D'] = dist.mapping_alias.join(",")
+ }
+ d['E'] = dist.address
+ }
+
+ def dist1 = CommonDatautils.findDistrict(d['A'], newRegion, d['G'], "district")
+ if(dist1){
+ if(dist1.mapping_alias){
+ d['H'] = dist1.mapping_alias.join(",")
+ }
+ d['I'] = dist1.address
+ }
+ }
+ }
+ });
+ }
+
def run4() {
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
def regionCode = null
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy
new file mode 100644
index 000000000..ebb9c165b
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/OracleHouse.groovy
@@ -0,0 +1,108 @@
+package com.fangjia.script.ccbevaluation.data
+
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.gj.ccb.FileUtils
+import com.fangjia.script.util.ElasticUtils
+import com.mongodb.BasicDBObject
+import groovy.sql.Sql
+import org.elasticsearch.index.query.BoolQueryBuilder
+import org.elasticsearch.index.query.QueryBuilder
+import org.elasticsearch.index.query.QueryBuilders
+import org.elasticsearch.xpack.sql.jdbc.EsDataSource
+
+import java.sql.ResultSet
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+
+class OracleHouse {
+ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
+ "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
+
+
+ public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
+ public static EsDataSource dataSource = new EsDataSource()
+ static {
+ dataSource.setUrl(address)
+ Properties connectionProperties = new Properties()
+ connectionProperties.put("user", "developer")
+ connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
+ dataSource.setProperties(connectionProperties)
+ }
+
+
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
+
+ static main(args) {
+ def task = new OracleHouse()
+ task.run()
+ }
+
+ def run(){
+ def rets = findAllHouse("2022-04-19")
+ def totalCount = new AtomicLong()
+ println rets.size()
+ final CountDownLatch latch = new CountDownLatch(rets.size())
+ for(def i=0;i
+ index = index + 1
+ println index
+ def data = []
+ data << r.FORSALEID
+ data << r.COMMID
+ allRows << data
+ });
+ return allRows
+ }
+
+ def removeHouse(houseid){
+ def sql = "delete from bigdata_forsale_gj where FORSALEID='${houseid}' ";
+ sqlInstance.executeUpdate(sql)
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy
index bae011454..9ac21fba3 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/MatchEvaluateRegionTest.groovy
@@ -2,39 +2,70 @@ package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
+import com.fangjia.core.util.DateUtil
+import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
+import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
+import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.GeoHelper
+import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.OpenApiClient
+import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
+import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
+import com.mongodb.DBObject
import groovy.sql.Sql
+import net.sf.json.JSONObject
+import org.apache.commons.lang.StringUtils
+import org.elasticsearch.index.query.QueryBuilder
+import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
+import java.util.zip.GZIPInputStream
class MatchEvaluateRegionTest {
+ static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
+ def path = "D:\\数据\\美团\\10城小区明细-V10.xlsx"
+ def total = 1500
+
static main(args) {
def task = new MatchEvaluateRegionTest()
- task.run()//小区匹配
- task.run1()//地图匹配
+// task.run()//小区匹配
+// task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
- task.addPrice()
- task.addAuthor()
+// task.addPrice()
+// task.addArroundPrice()
+// task.addArroundPriceByLat()
+// task.addRentRate()
+// task.addAuthor()
+// task.addLevel()
+// task.addSorce()
+// task.addArroundScore()
+
+ task.pkPrice()
+
}
def run(){
- def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
+
+ def index = 0
ExcelUtils.multiThreadingModify(path, { row ->
+ index ++
if(!row['F']){
- def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null)
+ println index+" "+row['E']
+ def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false,row['B'],null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
@@ -43,18 +74,17 @@ class MatchEvaluateRegionTest {
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
-// row['K'] = jsonObject.get("lng")
-// row['L'] = jsonObject.get("lat")
+ row['M'] = jsonObject.get("lng")
+ row['N'] = jsonObject.get("lat")
}
}
});
}
def run1(){
- def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
- if(!row['F']){
- def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null)
+ if(!row['F'] && row['A']){
+ def jsonObject = OpenApiClient.addressMatch(row['A'], row['E'], null, true, row['B'],row['C'].toString())
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
@@ -63,52 +93,104 @@ class MatchEvaluateRegionTest {
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
-// row['K'] = jsonObject.get("lng")
-// row['L'] = jsonObject.get("lat")
+ row['M'] = jsonObject.get("lng")
+ row['N'] = jsonObject.get("lat")
}
}
});
}
+ def pkPrice(){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ if(row['R'] && row['R'].toString().toDouble().toInteger()==1){
+ def obj = CommonDatautils.queryDistrictPrice(row['G'], row['H'], row['I'], DateUtils.parseFromTightByPattern("2022-05-22","yyyy-MM-dd"))
+ if(!obj){
+ row['R'] = ''
+ row['S'] = "无最新价"
+ }
+ }
+ });
+
+ }
+
def addPrice(){
- def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['L']){
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
if(priceObj){
- row['M'] = priceObj.avg_price
- row['N']= (int)(priceObj.avg_price * row['E'])
+ row['O'] = priceObj.avg_price
+ // row['N']= (int)(priceObj.avg_price * row['E'])
}
- def avg_price = findAvgPrice(row['M'])
- if(row['M'] && avg_price){
- def weekendPrice = row['M'].toString().toDouble()
- def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
- row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
+ def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['G'], row['H'], row['I'])
+ if(sqmetreRentPrice){
+ row['P'] = sqmetreRentPrice
}
+
+// def avg_price = findAvgPrice(row['F'])
+// if(row['M'] && avg_price){
+// def weekendPrice = row['M'].toString().toDouble()
+// def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
+// row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
+// }
}
});
}
+ def addRentRate(){
+ def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
+ def radioMap = [:]
+ for(def city : citys) {
+ def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
+ def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
+ radioMap.put(city, avg_radio)
+ }
+
+ ExcelUtils.multiThreadingModify(path, { row ->
+ def city = row['G']
+ if (row['F'] && row['O'] && !row['P']) {
+ def avg_rent_radio = radioMap.get(city)
+ if(avg_rent_radio){
+ row['P'] = (int)(row['O']/avg_rent_radio)
+ row['S'] = "租金推算1"
+ }
+ }
+ })
+ }
+
+ def addArroundPrice(){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ if(!row['O']) {
+ def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['E'])
+ if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
+ def priceObj = arroundObj.result
+ row['O'] = priceObj.unitPrice
+// row['N'] = (int) (priceObj.unitPrice * row['E'])
+ row['S'] = "周边价格"
+ }
+ }
+ });
+ }
+
def addAuthor(){
- def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
- def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
- def total = 747
+ def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
+ if(row['B']){
if(index < avg){
- row['P'] = authors[count]
+ row['N'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
+ }
});
}
@@ -236,4 +318,295 @@ class MatchEvaluateRegionTest {
});
return avg_price
}
+
+ def addSorce(){
+ ExcelUtils.multiThreadingModify(path, { d ->
+ if(d['F']){
+ def score = findDistrictScore(d['F'])
+ if(score){
+ d['R'] = score
+ }
+ }
+ });
+ }
+
+ def addArroundScore(){
+ ExcelUtils.multiThreadingModify(path, { d ->
+ if(!d['R'] && d['F']){
+ def dist = null
+ if(d['F'].toString().size()>16){
+ dist = CommonDatautils.findDistrictById(d['F'])
+ }
+
+ if(!dist){
+ dist = new BasicDBObject()
+ dist.city = d['G']
+ dist.lng = d['M'].toString().toDouble()*10000000000L
+ dist.lat = d['N'].toString().toDouble()*10000000000L
+ }
+ if(dist.lat){
+ List list = DbUtils.findAroundDistrict(dist)
+ if (list) {
+ list = sortByDistance(dist, list);
+ if(list && list.size()>10){
+ list = list[0..9]
+ }
+ if(list){
+ def scores = []
+ for(def s : list){
+ def score = findDistrictScore(s._id.toString())
+ if(score){
+ scores << score
+ }
+ }
+ def arround_score =(int)(scores.sum()/scores.size())
+ d['R'] = arround_score
+ d['T'] = "周边小区评分2"
+ }
+ }
+ }
+ }
+ });
+ }
+
+ def addArroundPriceByLat(){
+ ExcelUtils.multiThreadingModify(path, { d ->
+ if(!d['O'] && d['F'] && (d['K'].contains("住宅") || d['K'].contains("商住") ||d['K'].contains("别墅"))){
+ def dist = null
+ if(d['F'].toString().size()>16){
+ dist = CommonDatautils.findDistrictById(d['F'])
+ }
+
+ if(!dist){
+ dist = new BasicDBObject()
+ dist.city = d['G']
+ dist.lng = d['M'].toString().toDouble()*10000000000L
+ dist.lat = d['N'].toString().toDouble()*10000000000L
+ }
+ if(dist.lat){
+ List list = DbUtils.findAroundDistrict(dist)
+ if (list) {
+ list = sortByDistance(dist, list);
+ if(list && list.size()>10){
+ list = list[0..9]
+ }
+ if(list){
+ def prices = []
+ for(def s : list){
+ def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
+ if(priceObj){
+ prices << priceObj.avg_price
+ }
+ }
+ if(prices){
+ def arround_price =(int)(prices.sum()/prices.size())
+ d['O'] = arround_price
+ d['S'] = "周边均价3"
+ }
+ }
+ }
+ }
+ }
+ });
+ }
+
+
+ def sortByDistance(d, list){
+ def newList = []
+ for(def nb : list){
+ def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
+ nb.distance = distance
+ newList << nb
+ }
+ //排序
+ Collections.sort(newList, new Comparator() {
+ @Override
+ int compare(Object o1, Object o2) {
+ if(o1.distance > o2.distance)
+ return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
+ if(o1.distance == o2.distance)
+ return 0;
+ return -1;
+ }
+ })
+
+ return newList;
+ }
+
+
+
+ def findDistrictScore(id){
+ QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
+ def indexObj = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
+ def score = 0
+ for(def h : indexObj){
+ score = h.total_price_rate
+ if(score){
+ break;
+ }
+ }
+ return score
+ }
+
+
+ def addLevel() {
+ def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
+ def weekend = DateUtils.getFirstDayOfWeek(new Date());
+ def cityLevelMap = [:]
+ for(def city : citys){
+ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
+ def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
+ def datas = addPriceData(dists, priceMap);
+ datas = calcuteLevel(datas);
+ cityLevelMap.put(city, ["price":priceMap, "level":datas])
+ println city +" 楼盘评级同步完成"
+ }
+
+ ExcelUtils.multiThreadingModify(path, { row ->
+
+ def address = row['E']
+ def city = RegionData.formatCity(row['A'])
+
+ if(row['K'] && !row['K'].contains("住宅") && !row['K'].contains("商住") && !row['K'].contains("别墅")){
+ if(city){
+ def map = cityLevelMap.get(city)
+ def rMap = null
+ if(map){
+ rMap = getArroundDistrictPriceByAddress(city, address, map.level)
+ }
+
+ if(rMap){
+ if(rMap.rank){
+ row['Q'] = rMap.rank
+ }
+ }
+ }
+ }
+ })
+ }
+
+ def addPriceData(dists, priceMap){
+ def newLists = []
+ for(def d : dists){
+ def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
+ if(price){
+ d.avg_price = price;
+ newLists << d;
+ }
+ }
+ if(newLists){
+ newLists.sort {a,b->
+ def v1 = a.avg_price
+ def v2 = b.avg_price
+ return v1.compareTo(v2)
+ }
+ }
+ return newLists;
+ }
+
+ def calcuteLevel(def datas){
+ def total = datas.size()
+ def v = (int)(total/10)
+ def start = 0;
+ def end = 0;
+ def map = [:]
+ for(int i=1;i<=10;i++){
+ end = end + v;
+ if(i == 10){
+ map.put(i, [start, total])
+ }else{
+ map.put(i, [start, end])
+ }
+ start = end;
+ }
+ for(def d=0;dm && d<=n){
+ dist.level = 10 - o.key + 1
+ break
+ }
+ }
+ }
+ }
+ return datas;
+ }
+
+ def getArroundDistrictPriceByAddress(def city, def address, def levels){
+ def json = GaoDeApiUtils.geo(city, address)
+ def location = null
+ def detail = new BasicDBObject()
+ def level = null
+ def rank = null
+ if(json && json.count>=1){
+ def m = json.geocodes[0]
+ if(m){
+ location = m.location
+ level = m.level
+ }
+ }
+ if(location){
+ def arrs = location.split(",")
+ detail.lng = arrs[0].toString().toDouble()*10000000000L
+ detail.lat = arrs[1].toString().toDouble()*10000000000L
+ }
+ if(detail.lng){
+ detail.city = city
+
+ List list = DbUtils.findAroundDistrictByRadius(detail, 3000)
+ if(list && list.size()>10){
+ list = sortByDistance(detail, list);
+ try{
+ list = list[0..9]
+ }catch(Exception e){
+ println list.size()
+ }
+
+ }
+ def arroundAvgPrice = getArroundAvgPrice(list)
+ if(arroundAvgPrice){
+
+ def mod = Integer.MAX_VALUE;
+ for(def d : levels){
+ def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
+ if(newMod
if(!row['D'] && row['B']){
- def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
+ def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
@@ -77,7 +77,7 @@ class MatchEvaluateTest {
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
- def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null)
+ def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
@@ -117,7 +117,7 @@ class MatchEvaluateTest {
def addAuthor(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
- def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
+ def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy
new file mode 100644
index 000000000..42aedde30
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/PocStat.groovy
@@ -0,0 +1,1221 @@
+package com.fangjia.script.chongqing
+
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.common.DataHandleUtils
+import com.fangjia.script.temp.shenjian.util.SeaweedUtils
+import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.fangjia.script.util.OpenApiClient
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.mongodb.QueryOperators
+
+import java.util.concurrent.CountDownLatch
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+import java.util.concurrent.atomic.AtomicLong
+
+class PocStat {
+
+
+ public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
+ public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
+ public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
+
+
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
+
+ static main(args) {
+ def task = new PocStat();
+// task.regionDistStat();
+// task.sampleStat()
+// task.tradeStat()
+// task.paimaiStat()
+// task.evalStat()
+// task.fileldStat()
+
+// task.removeDelete()
+
+// task.loudongStat()
+// task.houseLoudongStat()
+
+ task.houseLoudongStat2()
+
+// task.updteHoledCount()
+ }
+
+ def loudongStat(){
+ def retMaps = getAllLouDong()
+ def total_count = retMaps.keySet().size()
+
+ def all_floorsCount = 0;
+ def all_buildYearCount = 0
+ def all_householdsCount = 0
+ def all_elevatorNumberCount = 0
+ def all_roomCount = 0
+
+ def f_all_floorsCount = 0;
+ def f_all_buildYearCount = 0
+ def f_all_householdsCount = 0
+ def f_all_elevatorNumberCount = 0
+ def f_all_roomCount = 0
+
+
+ def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
+
+ def z_count = 0
+ def f_count = 0
+
+ retMaps.each{key,lists->
+ def floorsCount = 0;
+ def buildYearCount = 0
+ def householdsCount = 0
+ def elevatorNumberCount = 0
+ def roomCount = 0
+
+ def f_floorsCount = 0;
+ def f_buildYearCount = 0
+ def f_householdsCount = 0
+ def f_elevatorNumberCount = 0
+ def f_roomCount = 0
+
+ def dist = CommonDatautils.findDistrictById(key)
+
+ if(zRegions.contains(dist.region)){
+ z_count++
+ }else{
+ f_count++
+ }
+
+ for(def row : lists){
+
+ if(zRegions.contains(dist.region)){
+
+ if(row.floors){
+ floorsCount++
+ }
+ if(dist.finish_building_date){
+ buildYearCount++
+ }
+ if(row.households){
+ householdsCount++
+ }else{
+ if(row.room_count && row.floors){
+ householdsCount++
+ }
+ }
+ if(row.elevatorNumber && row.elevatorNumber>0){
+ elevatorNumberCount++
+ }
+
+ if(row.room_count>0){
+ roomCount++
+ }
+
+ }else{
+ if(row.floors){
+ f_floorsCount++
+ }
+ if(dist.finish_building_date){
+ f_buildYearCount++
+ }
+ if(row.households){
+ f_householdsCount++
+ }else{
+ if(row.room_count && row.floors){
+ f_householdsCount++
+ }
+ }
+ if(row.elevatorNumber && row.elevatorNumber>0){
+ f_elevatorNumberCount++
+ }
+
+ if(row.room_count>0){
+ f_roomCount++
+ }
+ }
+
+ }
+
+ def dist_total_count = lists.size()*0.6
+ if(floorsCount >= dist_total_count){
+ all_floorsCount ++
+ }
+
+ if(buildYearCount >= dist_total_count){
+ all_buildYearCount ++
+ }
+
+ if(householdsCount >= dist_total_count){
+ all_householdsCount ++
+ }
+
+ if(elevatorNumberCount >= dist_total_count){
+ all_elevatorNumberCount ++
+ }
+
+ if(roomCount >= dist_total_count){
+ all_roomCount ++
+ }
+
+
+ if(f_floorsCount >= dist_total_count){
+ f_all_floorsCount ++
+ }
+
+ if(f_buildYearCount >= dist_total_count){
+ f_all_buildYearCount ++
+ }
+
+ if(f_householdsCount >= dist_total_count){
+ f_all_householdsCount ++
+ }
+
+ if(f_elevatorNumberCount >= dist_total_count){
+ f_all_elevatorNumberCount ++
+ }
+ if(f_roomCount >= dist_total_count){
+ f_all_roomCount ++
+ }
+ }
+ def floor_prec = DataHandleUtils.getDoubleRound(2, all_floorsCount*100/z_count)
+ def year_prec = DataHandleUtils.getDoubleRound(2, all_buildYearCount*100/z_count)
+ def households_prec = DataHandleUtils.getDoubleRound(2, all_householdsCount*100/z_count)
+ def elevatorNum_prec = DataHandleUtils.getDoubleRound(2, all_elevatorNumberCount*100/z_count)
+ def roomcount_prec = DataHandleUtils.getDoubleRound(2, all_roomCount*100/z_count)
+
+
+ def f_floor_prec = DataHandleUtils.getDoubleRound(2, f_all_floorsCount*100/f_count)
+ def f_year_prec = DataHandleUtils.getDoubleRound(2, f_all_buildYearCount*100/f_count)
+ def f_households_prec = DataHandleUtils.getDoubleRound(2, f_all_householdsCount*100/f_count)
+ def f_elevatorNum_prec = DataHandleUtils.getDoubleRound(2, f_all_elevatorNumberCount*100/f_count)
+ def f_roomcount_prec = DataHandleUtils.getDoubleRound(2, f_all_roomCount*100/f_count)
+
+ println "floor "+ floor_prec
+ println "year "+ year_prec
+ println "households "+ households_prec
+ println "elevator "+ elevatorNum_prec
+
+ def rets = []
+ rets << ["城区","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观"]
+ rets << ["主城区",100,100,100,0,floor_prec,0,year_prec,elevatorNum_prec,roomcount_prec,households_prec,0]
+ rets << ["非主城区",100,100,100,0,f_floor_prec,0,f_year_prec,f_elevatorNum_prec,f_roomcount_prec,f_households_prec,0]
+ ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "楼栋")
+ }
+
+
+ def houseLoudongStat2(){
+ def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
+ def dists = CommonDatautils.findAllDistrictByCity("重庆","district", ["city","region","name"])
+ def z_count = 0
+ def f_count = 0
+
+ def z_directionCount = 0
+ def f_directionCount = 0
+
+ def f_sightCount = 0
+ def z_sightCount = 0
+ def index = 0
+ for(def dist : dists){
+ index ++
+ def rets = getAllHouseLoudong2(dist._id.toString())
+ println index +" "+rets.size()
+ if(rets){
+ def directionCount = 0
+ def sightCount = 0
+ for(def d : rets){
+ if(d.direction){
+ directionCount ++
+ }
+ if(d.sight){
+ sightCount ++
+ }
+ }
+
+ def total = rets.size()*0.6
+ if(zRegions.contains(dist.region)){
+ z_count++
+
+ if(directionCount>=total){
+ z_directionCount++
+ }
+ if(sightCount >=total){
+ z_sightCount++
+ }
+ }else{
+ f_count++
+
+ if(directionCount>=total){
+ f_directionCount++
+ }
+ if(sightCount >=total){
+ f_sightCount++
+ }
+ }
+ }
+ }
+
+ def z_direct_prec = DataHandleUtils.getDoubleRound(2, z_directionCount*100/z_count)
+ def z_sight_prec = DataHandleUtils.getDoubleRound(2, z_sightCount*100/z_count)
+
+ def f_direct_prec = DataHandleUtils.getDoubleRound(2, f_directionCount*100/f_count)
+ def f_sight_prec = DataHandleUtils.getDoubleRound(2, f_sightCount*100/f_count)
+ def rets = []
+ rets << ["城区","户号","户ID","主朝向","景观","户型结构"]
+ rets << ["主城区",100,100,z_direct_prec, z_sight_prec, 100]
+ rets << ["非主城区",100,100, f_direct_prec, f_sight_prec, 100]
+ ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "户1")
+ }
+
+ def houseLoudongStat(){
+ def map = getAllHouseLoudong()
+ def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
+
+ def z_count = 0
+ def f_count = 0
+
+ def z_directionCount = 0
+ def f_directionCount = 0
+
+ def f_sightCount = 0
+ def z_sightCount = 0
+ map.each{key,list->
+ def dist = CommonDatautils.findDistrictById(key)
+ def directionCount = 0
+ def sightCount = 0
+ for(def d : list){
+ if(d.direction){
+ directionCount ++
+ }
+ if(d.sight){
+ sightCount ++
+ }
+ }
+
+ def total = list.size()*0.6
+ if(zRegions.contains(dist.region)){
+ z_count++
+
+ if(directionCount>=total){
+ z_directionCount++
+ }
+ if(sightCount >=total){
+ z_sightCount++
+ }
+ }else{
+ f_count++
+
+ if(directionCount>=total){
+ f_directionCount++
+ }
+ if(sightCount >=total){
+ f_sightCount++
+ }
+ }
+ }
+
+ def z_direct_prec = DataHandleUtils.getDoubleRound(2, z_directionCount*100/z_count)
+ def z_sight_prec = DataHandleUtils.getDoubleRound(2, z_sightCount*100/z_count)
+
+ def f_direct_prec = DataHandleUtils.getDoubleRound(2, f_directionCount*100/f_count)
+ def f_sight_prec = DataHandleUtils.getDoubleRound(2, f_sightCount*100/f_count)
+ def rets = []
+ rets << ["城区","户号","户ID","主朝向","景观","户型结构"]
+ rets << ["主城区",100,100,z_direct_prec, z_sight_prec, 100]
+ rets << ["非主城区",100,100, f_direct_prec, f_sight_prec, 100]
+ ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rets, "户")
+ }
+
+ def getAllHouseLoudong(){
+ def q = new BasicDBObject()
+ q.put("city","重庆")
+ q.put("district_id", new BasicDBObject(QueryOperators.EXISTS, true))
+
+ def keys = new BasicDBObject()
+ keys.put("district_id",1)
+ keys.put("direction",1)
+ keys.put("sight",1)
+ def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
+ def retMap = [:]
+ for(def d : dbcursor){
+ def districtId = d.district_id
+ if(districtId){
+ def lists = retMap.get(districtId)
+ if(!lists){
+ lists = []
+ retMap.put(districtId, lists)
+ }
+ lists << d
+ }
+ }
+ return retMap;
+ }
+
+
+ def getAllHouseLoudong2(districtId){
+ def q = new BasicDBObject()
+ q.put("city","重庆")
+ q.put("district_id", districtId)
+
+ def keys = new BasicDBObject()
+ keys.put("district_id",1)
+ keys.put("direction",1)
+ keys.put("sight",1)
+ def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
+ def lists = []
+ for(def d : dbcursor){
+ lists << d
+ }
+ return lists;
+ }
+
+ def getAllLouDong(){
+ def q = new BasicDBObject()
+ q.put("city","重庆")
+ q.put("id_in_seaweed", new BasicDBObject(QueryOperators.EXISTS, true))
+ def dbcursor = LOUDONG_LINE_COLL.find(q).batchSize(5000)
+ def retMap = [:]
+ for(def d : dbcursor){
+ def districtId = d.id_in_seaweed
+ if(districtId){
+ def lists = retMap.get(districtId)
+ if(!lists){
+ lists = []
+ retMap.put(districtId, lists)
+ }
+ lists << d
+ }
+ }
+ return retMap;
+ }
+
+ def updteHoledCount(){
+ def datas = findAllLouDong()
+ def totalCount = new AtomicLong()
+ println datas.size()
+ final CountDownLatch latch = new CountDownLatch(datas.size())
+ for(def i=0;i2){
+ q.put("unit_num", arrs[2])
+ }
+
+ def udata = new BasicDBObject()
+ udata.put("room_count", max_room)
+ LOUDONG_LINE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
+ }
+ } catch (Exception e) {
+ e.printStackTrace()
+ } finally {
+ latch.countDown()
+ }
+ };
+ })
+ }
+ latch.await()
+ }
+
+ def findAllLouDong(){
+ def q = new BasicDBObject()
+ q.put("city","重庆")
+ q.put("id_in_seaweed", new BasicDBObject(QueryOperators.EXISTS, true))
+ def dbcursor = LOUDONG_LINE_COLL.find(q)
+ def keys = new HashSet()
+ for(def d : dbcursor){
+ def districtId = d.id_in_seaweed
+ def key = districtId +"##"+d.house_num
+ if(d.unit_num && d.unit_num!="0"){
+ key = key + "##"+d.unit_num
+ }
+ if(!keys.contains(key)){
+ keys << key
+ }
+ }
+ return keys
+ }
+
+ def findHouseLouDongRoom(def arrs){
+ def q = new BasicDBObject()
+ q.put("city","重庆")
+ q.put("district_id", arrs[0])
+ q.put("house_num", arrs[1])
+ if(arrs.size()>2){
+ q.put("unit_num", arrs[2])
+ }
+ def dbcursor = HOUSE_LOUDONG_COLL.find(q)
+ def max_room = 0
+ for(def d : dbcursor){
+ def room = null
+ if(d.room_num && d.room_num.contains("-")){
+ room = d.room_num.substring(d.room_num.indexOf("-")+1, d.room_num.length())
+ }else if(d.room_num && d.room_num.contains("0")){
+ room = d.room_num.substring(d.room_num.toString().lastIndexOf("0")+1, d.room_num.length())
+ }
+ if(room && room.isNumber()){
+ if(room.toInteger()>max_room){
+ max_room = room.toInteger()
+ }
+ }
+ }
+ return max_room;
+ }
+ def removeDelete(){
+ def dists = CommonDatautils.findAllDistrictByCity("重庆","district", ["city","region","name", "name_cat"])
+ for(def obj : dists){
+ def name_cat = obj.name_cat;
+ if(name_cat==12){
+ def count = CommonDatautils.findDistrictCount(obj.city, obj.region, obj.name, "district")
+ if(count>1){
+ println obj.name +" "+count
+ DbUtils.seaweed.remove(new BasicDBObject("_id", obj._id))
+ }
+ }
+ }
+ }
+
+ def regionDistStat(){
+ def city = "重庆"
+ def regionMap = CommonDatautils.getRegionsByCity(city)
+ def set =regionMap.keySet();
+ def rows = []
+ rows << ["区域","小区量"]
+ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
+ for(def region : set){
+ def count = CommonDatautils.findDistrictCountByRegionBlock(city, region, null, "district")
+
+// def regionCount = 0
+// def regionHouseLDCount = 0
+// for(def d : dists){
+// if(d.region == region){
+// def loudongCount = CommonDatautils.findLoudong(d._id.toString())
+// if(loudongCount && loudongCount>0){
+// regionCount = regionCount+ loudongCount
+// }
+//
+// def houseLouDongCount = CommonDatautils.findHouseLoudong(d._id.toString())
+// if(houseLouDongCount && houseLouDongCount>0){
+// regionHouseLDCount = regionHouseLDCount + houseLouDongCount
+// }
+// }
+// }
+ rows << [region, count]
+ println region +" 完成"
+ }
+ ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx",rows, "各区楼盘量统计1")
+ }
+
+ def sampleStat(){
+ def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
+
+ final CountDownLatch latch = new CountDownLatch(dists.size())
+
+ def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
+ def zCityDistTotal = new AtomicLong()
+ def fCityDistTotal = new AtomicLong()
+ //[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
+ def zHLevel1 = new AtomicLong()
+ def zHLevel2 = new AtomicLong()
+ def zHLevel3 = new AtomicLong()
+ def zHLevel4 = new AtomicLong()
+ def zHLevel5 = new AtomicLong()
+
+ def fHLevel1 = new AtomicLong()
+ def fHLevel2 = new AtomicLong()
+ def fHLevel3 = new AtomicLong()
+ def fHLevel4 = new AtomicLong()
+ def fHLevel5 = new AtomicLong()
+
+ def rets = []
+ rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
+
+
+ def index = new AtomicLong()
+ for(def i=0;i=50){
+ zHLevel5.addAndGet(1)
+ }
+ }
+ }else{
+ fCityDistTotal.addAndGet(1)
+
+ if(results){
+ if(results.size()<5){
+ fHLevel2.addAndGet(1)
+ }else if(results.size()<20){
+ fHLevel3.addAndGet(1)
+ }else if(results.size()<50){
+ fHLevel4.addAndGet(1)
+ }else if(results.size()>=50){
+ fHLevel5.addAndGet(1)
+ }
+ }
+ }
+ }else{
+ if(zRegions.contains(d.region)) {
+ zCityDistTotal.addAndGet(1)
+ zHLevel1.addAndGet(1)
+
+ println d.city+"#"+d.region+"#"+d.name
+ }else{
+ fCityDistTotal.addAndGet(1)
+ fHLevel1.addAndGet(1)
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace()
+ } finally {
+ latch.countDown()
+ }
+ }
+ })
+ }
+ latch.await()
+ def ret1 = ["主城区", zCityDistTotal.get(), zHLevel1.get(), zHLevel2.get(), zHLevel3.get(), zHLevel4.get(), zHLevel5.get()]
+ def ret2 = ["非主城区", fCityDistTotal.get(), fHLevel1.get(), fHLevel2.get(), fHLevel3.get(), fHLevel4.get(), fHLevel5.get()]
+ rets.add(ret1)
+ rets.add(ret2)
+ ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-样本")
+ }
+
+
+ def tradeStat(){
+ def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
+
+ final CountDownLatch latch = new CountDownLatch(dists.size())
+
+ def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
+ def zCityDistTotal = new AtomicLong()
+ def fCityDistTotal = new AtomicLong()
+ //[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
+ def zTLevel1 = new AtomicLong()
+ def zTLevel2 = new AtomicLong()
+ def zTLevel3 = new AtomicLong()
+ def zTLevel4 = new AtomicLong()
+ def zTLevel5 = new AtomicLong()
+
+ def fTLevel1 = new AtomicLong()
+ def fTLevel2 = new AtomicLong()
+ def fTLevel3 = new AtomicLong()
+ def fTLevel4 = new AtomicLong()
+ def fTLevel5 = new AtomicLong()
+
+ def rets = []
+ rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
+
+
+ def index = new AtomicLong()
+ for(def i=0;i=50){
+ zTLevel5.addAndGet(1)
+ }
+ }
+ }else{
+ fCityDistTotal.addAndGet(1)
+
+ if(results){
+ if(results.size()<5){
+ fTLevel2.addAndGet(1)
+ }else if(results.size()<20){
+ fTLevel3.addAndGet(1)
+ }else if(results.size()<50){
+ fTLevel4.addAndGet(1)
+ }else if(results.size()>=50){
+ fTLevel5.addAndGet(1)
+ }
+ }
+ }
+ }else{
+ if(zRegions.contains(d.region)) {
+ zCityDistTotal.addAndGet(1)
+ zTLevel1.addAndGet(1)
+ }else{
+ fCityDistTotal.addAndGet(1)
+ fTLevel1.addAndGet(1)
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace()
+ } finally {
+ latch.countDown()
+ }
+ }
+ })
+ }
+ latch.await()
+ def ret1 = ["主城区", zCityDistTotal.get(), zTLevel1.get(), zTLevel2.get(), zTLevel3.get(), zTLevel4.get(), zTLevel5.get()]
+ def ret2 = ["非主城区", fCityDistTotal.get(), fTLevel1.get(), fTLevel2.get(), fTLevel3.get(), fTLevel4.get(), fTLevel5.get()]
+ rets.add(ret1)
+ rets.add(ret2)
+ ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-成交")
+ println "成交案例统计完成"
+ }
+
+ def paimaiStat(){
+ def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
+
+ final CountDownLatch latch = new CountDownLatch(dists.size())
+
+ def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
+ def zCityDistTotal = new AtomicLong()
+ def fCityDistTotal = new AtomicLong()
+ //[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
+ def zPLevel1 = new AtomicLong()
+ def zPLevel2 = new AtomicLong()
+ def zPLevel3 = new AtomicLong()
+ def zPLevel4 = new AtomicLong()
+ def zPLevel5 = new AtomicLong()
+
+ def fPLevel1 = new AtomicLong()
+ def fPLevel2 = new AtomicLong()
+ def fPLevel3 = new AtomicLong()
+ def fPLevel4 = new AtomicLong()
+ def fPLevel5 = new AtomicLong()
+
+ def rets = []
+ rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
+
+
+ def index = new AtomicLong()
+ for(def i=0;i=50){
+ zPLevel5.addAndGet(1)
+ }
+ }
+ }else{
+ fCityDistTotal.addAndGet(1)
+
+ if(results){
+ if(results.size()<5){
+ fPLevel2.addAndGet(1)
+ }else if(results.size()<20){
+ fPLevel3.addAndGet(1)
+ }else if(results.size()<50){
+ fPLevel4.addAndGet(1)
+ }else if(results.size()>=50){
+ fPLevel5.addAndGet(1)
+ }
+ }
+ }
+ }else{
+ if(zRegions.contains(d.region)) {
+ zCityDistTotal.addAndGet(1)
+ zPLevel1.addAndGet(1)
+ }else{
+ fCityDistTotal.addAndGet(1)
+ fPLevel1.addAndGet(1)
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace()
+ } finally {
+ latch.countDown()
+ }
+ }
+ })
+ }
+ latch.await()
+ def ret1 = ["主城区", zCityDistTotal.get(), zPLevel1.get(), zPLevel2.get(), zPLevel3.get(), zPLevel4.get(), zPLevel5.get()]
+ def ret2 = ["非主城区", fCityDistTotal.get(), fPLevel1.get(), fPLevel2.get(), fPLevel3.get(), fPLevel4.get(), fPLevel5.get()]
+ rets.add(ret1)
+ rets.add(ret2)
+ ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-拍卖")
+ println "拍卖案例统计完成"
+ }
+ def evalStat(){
+ def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
+
+ final CountDownLatch latch = new CountDownLatch(dists.size())
+
+ def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
+ def zCityDistTotal = new AtomicLong()
+ def fCityDistTotal = new AtomicLong()
+ //[0, 1), [1, 5), [5, 20), [20, 50), [50,+)
+ def zELevel1 = new AtomicLong()
+ def zELevel2 = new AtomicLong()
+ def zELevel3 = new AtomicLong()
+ def zELevel4 = new AtomicLong()
+ def zELevel5 = new AtomicLong()
+
+ def fELevel1 = new AtomicLong()
+ def fELevel2 = new AtomicLong()
+ def fELevel3 = new AtomicLong()
+ def fELevel4 = new AtomicLong()
+ def fELevel5 = new AtomicLong()
+
+ def rets = []
+ rets << ["城区","小区量","[0, 1)","[1, 5)","[5, 20)","[20, 50)","[50,+)"]
+
+
+ def index = new AtomicLong()
+ for(def i=0;i=50){
+ zELevel5.addAndGet(1)
+ }
+ }
+ }else{
+ fCityDistTotal.addAndGet(1)
+
+ if(results){
+ if(results.size()<5){
+ fELevel2.addAndGet(1)
+ }else if(results.size()<20){
+ fELevel3.addAndGet(1)
+ }else if(results.size()<50){
+ fELevel4.addAndGet(1)
+ }else if(results.size()>=50){
+ fELevel5.addAndGet(1)
+ }
+ }
+ }
+ }else{
+ if(zRegions.contains(d.region)) {
+ zCityDistTotal.addAndGet(1)
+ zELevel1.addAndGet(1)
+ }else{
+ fCityDistTotal.addAndGet(1)
+ fELevel1.addAndGet(1)
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace()
+ } finally {
+ latch.countDown()
+ }
+ }
+ })
+ }
+ latch.await()
+ def ret1 = ["主城区", zCityDistTotal.get(), zELevel1.get(), zELevel2.get(), zELevel3.get(), zELevel4.get(), zELevel5.get()]
+ def ret2 = ["非主城区", fCityDistTotal.get(), fELevel1.get(), fELevel2.get(), fELevel3.get(), fELevel4.get(), fELevel5.get()]
+ rets.add(ret1)
+ rets.add(ret2)
+ ExcelUtils.write("D:\\数据\\重庆农商行\\poc\\底层数据统计.xlsx", rets, "案例数量分布-评估")
+ println "评估案例统计完成"
+ }
+
+ def fileldStat(){
+ def dists = CommonDatautils.findAllDistrictByCity("重庆", "district", ["city","region","name"])
+ final CountDownLatch latch = new CountDownLatch(dists.size())
+ def zRegions = ["渝中","大渡口","江北","沙坪坝","九龙坡","南岸","北碚","渝北","巴南"]
+ def alias_count = new AtomicLong()
+ def address_count = new AtomicLong()
+ def block_count = new AtomicLong()
+ def township_count = new AtomicLong()
+ def price_count = new AtomicLong()
+ def lng_count = new AtomicLong()
+ def finishBuildTime_count = new AtomicLong()
+ def developer_count = new AtomicLong()
+ def green_count = new AtomicLong()
+ def ratio_count = new AtomicLong()
+ def carsite_count = new AtomicLong()
+ def propertyManagement_count = new AtomicLong()
+ def managementFee_count = new AtomicLong()
+ def building_count = new AtomicLong()
+ def holed_count = new AtomicLong()
+ def car_holed_count = new AtomicLong()
+ def all_area_count = new AtomicLong()
+ def key_school_count = new AtomicLong()
+ def school_count = new AtomicLong()
+ def traffic_count = new AtomicLong()
+ def market_count = new AtomicLong()
+ def hospital_count = new AtomicLong()
+ def bank_count = new AtomicLong()
+
+ def f_alias_count = new AtomicLong()
+ def f_address_count = new AtomicLong()
+ def f_block_count = new AtomicLong()
+ def f_township_count = new AtomicLong()
+ def f_price_count = new AtomicLong()
+ def f_lng_count = new AtomicLong()
+ def f_finishBuildTime_count = new AtomicLong()
+ def f_developer_count = new AtomicLong()
+ def f_green_count = new AtomicLong()
+ def f_ratio_count = new AtomicLong()
+ def f_carsite_count = new AtomicLong()
+ def f_propertyManagement_count = new AtomicLong()
+ def f_managementFee_count = new AtomicLong()
+ def f_building_count = new AtomicLong()
+ def f_holed_count = new AtomicLong()
+ def f_car_holed_count = new AtomicLong()
+ def f_all_area_count = new AtomicLong()
+ def f_key_school_count = new AtomicLong()
+ def f_school_count = new AtomicLong()
+ def f_traffic_count = new AtomicLong()
+ def f_market_count = new AtomicLong()
+ def f_hospital_count = new AtomicLong()
+ def f_bank_count = new AtomicLong()
+
+ def zCityDistTotal = new AtomicLong()
+ def fCityDistTotal = new AtomicLong()
+ def index = new AtomicLong()
+ for(def i=0;i
+ if(row['F'] && row['R']){
+ def avg_price = findAvgPrice(row['R'])
+ if(avg_price){
+ def ppe15 = Math.abs((row['F'] - avg_price) * 100 / avg_price)
+ row['Z'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
+ }
+ }
+ });
+ }
+
+ def pkPrice(path){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ def commid = row['R']
+ if(commid){
+ def dist = CommonDatautils.findDistrictById(commid.trim())
+ if(dist){
+ def jsonObject = OpenApiClient.webankEvaluate(dist.city, dist.region, dist.name, row['E'].toString())
+ if(jsonObject != null && jsonObject.code==200) {
+ def result = jsonObject.result
+ if(result){
+ row['Y'] = result.get("unitPrice")
+ }
+ }
+ }
+ }
+ })
+ }
+
+
+ def match(path){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ if(!row['Q'] && row['D']){
+ def city = row['B']
+ def address = row['D']
+ def region = row['C']
+ println "[match] city:"+city+" region:"+region+" address:"+address
+ def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
+ if(jsonObject != null && jsonObject.precise!=-1) {
+ row['Q'] = jsonObject.get("name")
+ row['R'] = jsonObject.get("id")
+ }
+ }
+ })
+ }
+
+ def matchMap(path){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ if(!row['Q'] && row['D']){
+ def city = row['B']
+ def address = row['D']
+ def region = row['C']
+ println "city:"+city+" region:"+region+" address:"+address
+ def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null)
+ if(jsonObject != null && jsonObject.precise!=-1) {
+ def propertyType = jsonObject.get("propertyType")
+ if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){
+ row['Q'] = jsonObject.get("name")
+ row['R'] = new ObjectId().toString()
+ row['W'] = propertyType
+
+ def arroundObj = FangjiaClientUtils.arroundPrice(city, address)
+ if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
+ def priceObj = arroundObj.result
+ row['F'] = priceObj.unitPrice
+ }
+ }
+
+ }
+ }
+ })
+ }
+
+ def addPrice(path){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ def commid = row['R']
+ if(commid){
+ def dist = CommonDatautils.findDistrictById(commid.trim())
+ if(dist){
+ row['Q'] = dist.name
+ def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
+ if(priceObj){
+ row['F'] = priceObj.avg_price
+ }
+ }
+ }
+ })
+ }
+
+
+
+ def addArroundPrice(path){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ if(!row['F']) {
+ def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
+ if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
+ def priceObj = arroundObj.result
+ row['F'] = priceObj.unitPrice
+ row['W'] = "周边均价"
+ }
+ }
+ });
+ }
+
+ def standard(path){
+ ExcelUtils.multiThreadingModify(path, { row ->
+ def city = row['B']
+ def address = row['D']
+ if(address){
+ println "[standard] address:"+address
+ def jsonObject = OpenApiClient.standard2Action(city, address)
+ if(jsonObject){
+ println jsonObject
+ row['G'] = jsonObject.get("province")
+ row['H'] = jsonObject.get("city")
+ row['I'] = jsonObject.get("district")
+ row['J'] = jsonObject.get("township")
+ row['K'] = jsonObject.get("street")
+ row['L'] = jsonObject.get("streetNumber")
+ row['M'] = jsonObject.get("buildingNumber")
+ row['N'] = jsonObject.get("unitNumber")
+ row['O'] = jsonObject.get("floor")
+ row['P'] = jsonObject.get("roomNumber")
+ }
+ }
+
+ })
+ }
+
+ def addLoudongNum(path){
+
+ ExcelUtils.multiThreadingModify(path, { row ->
+ def commid = row['R']
+ def houseNum = row['M']
+ def unitNum = row['N']
+ println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum
+ if (commid && houseNum) {
+ def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum)
+ if (jsonObject && jsonObject.result) {
+ if(jsonObject.result.size()>=1){
+ def obj = jsonObject.result[0]
+ row['T'] = obj.id
+ row['S'] = obj.houseNumber
+ }
+ }
+
+ if(row['P']){
+ def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P'])
+ if (jsonObject1 && jsonObject1.result) {
+ if(jsonObject1.result.size()>=1){
+ def obj = jsonObject1.result[0]
+ row['V'] = obj.id
+ row['U'] = obj.roomNum
+ }
+ }
+ }
+ }
+
+ })
+ }
+
+ def createLoudong(path, output){
+ def rets = new CopyOnWriteArrayList()
+ def datas = ExcelUtils.read(path, 0, 1)
+ rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
+ for(def row : datas){
+ def data = []
+ data << row['B']
+ data << row['C']
+ data << row['D']
+ data << row['E']
+ if (row['R'] && row['M']) {
+ def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N'])
+ if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
+ def obj = jsonObject.result[0]
+ data << obj.houseNumber
+ data << obj.id
+ data << obj.type// 物业类型
+ data << ""//主朝向
+ data << obj.floors//地上层数
+ data << ""//地下层数
+ data << obj.buildYear;//竣工日期
+ data << ""//梯
+ data << ""//户
+ data << obj.households//总户数
+ data << ""//景观
+ }else{
+ data << ""
+ data << ""
+ data << ""// 物业类型
+ data << ""//主朝向
+ data << ""//地上层数
+ data << ""//地下层数
+ data << ""//竣工日期
+ data << ""//梯
+ data << ""//户
+ data << ""//总户数
+ data << ""//景观
+ }
+
+ if(row['P']){
+ def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P'])
+ if (jsonObject1 && jsonObject1.result) {
+ if(jsonObject1.result.size()>=1){
+ def obj = jsonObject1.result[0]
+
+ data << obj.roomNum//户号
+ data << obj.id//户ID
+ data << obj.direction//主
+ data << obj.sight//景观
+ if(obj.roomNum && obj.roomNum.contains("跃")){
+ data << "跃式"//户型结构
+ }else{
+ data << ""//户型结构
+ }
+
+ }
+ }
+ }
+ }
+ rets << data
+ }
+ ExcelUtils.write(output, rets, "底层数据-楼栋-户")
+ }
+
+ def addDetail(path, output){
+ def datas = ExcelUtils.read(path, 0, 1)
+ def rows = []
+ rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
+ for(def d : datas){
+ def commid = d['R']
+ println "[addDetail] commid:"+commid
+ if(commid){
+ def dist = CommonDatautils.findDistrictById(commid)
+ if(dist){
+ def row = []
+ def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
+ if(jsonObject) {
+
+ row << d['B']
+ row << d['C']
+ row << d['D']
+ row << jsonObject.get("name")//小区名称
+ row << commid//小区ID
+ if(jsonObject.get("alias")){
+ row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
+ }else{
+ row << ""
+ }
+
+ row << jsonObject.get("address")//小区地址
+ row << jsonObject.get("commercialArea")//板块
+ row << jsonObject.get("township")//街道***********
+ row << jsonObject.get("avgPrice")//基准价
+ row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度
+ row << jsonObject.get("finishBuildTime")//竣工时间
+ row << jsonObject.get("developer")//开发商名称
+ row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
+ row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
+ row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
+ row << jsonObject.get("propertyManagement")//物业公司
+ row << jsonObject.get("managementFee")//物业费
+ row << jsonObject.get("buildingCount")//住宅幢数
+ row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
+ row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
+ row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
+
+ }
+
+ def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
+ if(facilityObj && facilityObj.result){
+ def result = facilityObj.result
+ if(result.keySchool){
+ row << '是'
+ }else {
+ row << ""
+ }
+
+ if(result.school){
+ row << result.school.join(",").replace('"',"")
+ }else {
+ row << ""
+ }
+
+ def bus = []
+ if(result.bus){
+ bus.addAll(result.bus)
+ }
+ if(result.metro){
+ bus.addAll(result.metro)
+ }
+
+ if(bus){
+ row << bus.join(",").replace('"',"")
+ }else{
+ row << ""
+ }
+
+ if(result.supermarket){
+ row << result.supermarket.join(",").replace('"',"")
+ }else{
+ row << ""
+ }
+
+ if(result.hospital){
+ row << result.hospital.join(",").replace('"',"")
+ }else{
+ row << ""
+ }
+
+ if(result.bank){
+ row << result.bank.join(",").replace('"',"")
+ }else{
+ row << ""
+ }
+ }
+ rows << row
+ }
+ }
+ }
+ ExcelUtils.write(output, rows, "底层数据-小区")
+ }
+
+ def createPaimai(path, output){
+ def datas = ExcelUtils.read(path, 0, 1)
+ def rows = []
+ rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
+ def keys = []
+ for(def d : datas){
+ def commid = d['R']
+ println "[createPaimai] commid:"+commid
+ if(commid) {
+ def dist = CommonDatautils.findDistrictById(commid)
+ if (dist) {
+ def key = dist.city+"#"+dist.region+"#"+dist.name
+ if(keys.contains(key)){
+ continue
+ }
+ keys << key
+ def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100)
+ if(jsonObject && jsonObject.result){
+ def results = jsonObject.result
+ for(def r: results){
+ def row = []
+ row << d['B']
+ row << d['C']
+ row << d['D']
+ row << r.matchName
+ row << r.address
+ row << r.houseNum
+ row << r.roomNum
+ row << r.area
+ if(r.startDate){
+ row << r.startDate.substring(0, r.startDate.indexOf("T"))
+ }else{
+ row << ""
+ }
+
+ row << r.count
+ row << r.status
+ row << r.tradeAvgPrice
+ row << r.tradePrice
+ row << r.id
+ rows << row
+ }
+ println dist.name +" "+results.size()
+ }
+ }
+ }
+ }
+ ExcelUtils.write(output, rows, "拍卖案例")
+ }
+
+ def createHouse(path, output){
+ def datas = ExcelUtils.read(path, 0, 1)
+ def rows = []
+ rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
+ def keys = []
+ for(def d : datas) {
+ def commid = d['R']
+// println "[createHouse] commid:"+commid
+ if (commid) {
+ def key = d['Q']
+ if (keys.contains(key)) {
+ continue
+ }
+ keys << key
+ def jsonObject = OpenApiClient.houseList(commid,"20210501",20)
+ if(jsonObject && jsonObject.result) {
+ def results = jsonObject.result
+ for (def r : results) {
+ def row = []
+ row << "挂牌"
+ row << r.districtName
+ if(r.estateType){
+ row << r.estateType
+ }else{
+ if(r.purpose){
+ row << r.purpose
+ }else{
+ row << ""
+ }
+ }
+
+ if(r.room){
+ def roomStr = r.room+"室"
+ if(r.hall){
+ roomStr = roomStr +r.hall+"厅"
+ }
+ row << roomStr
+ }else{
+ row << ""
+ }
+ row << r.area
+ row << r.fitment
+ row << r.pDate
+ row << ""
+ row << r.avgPrice
+ row << r.totalPrice
+ if(r.floor){
+ row << r.floor
+ }else{
+ row << ""
+ }
+ if(r.height){
+ row << r.height
+ }else{
+ row << ""
+ }
+
+ row << r.direction
+ row << SourceNameUtil.getResourceName(r.root)
+ rows << row
+ }
+
+ println d['R'] +" "+results.size()
+ }
+ }
+ }
+ ExcelUtils.write(output, rows, "挂牌案例")
+ }
+
+ def createEvaluation(path, output){
+ def datas = ExcelUtils.read(path, 0, 1)
+ def rows = []
+ rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
+ def keys = []
+ for(def d : datas) {
+ def commid= d['R']
+ println "[createEvaluation] commid:"+commid
+ if (commid) {
+ if (keys.contains(commid)) {
+ continue
+ }
+ keys << commid
+ def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20)
+ if(jsonObject && jsonObject.result) {
+ def results = jsonObject.result
+ for (def r : results) {
+ def row = []
+ row << "评估"
+ row << r.commName
+ row << ""
+
+ if(r.roomcnt){
+ def roomStr = r.roomcnt+"室"
+ if(r.livingroomcnt){
+ roomStr = roomStr +r.livingroomcnt+"厅"
+ }
+ row << roomStr
+ }else{
+ row << ""
+ }
+ row << r.buildArea
+ row << r.fitment
+ row << r.cDate.substring(0, r.cDate.indexOf("T"))
+ row << ""
+ row << r.unitPrice
+ row << r.totalPrice
+ if(r.floor){
+ row << r.floor
+ }else{
+ row << ""
+ }
+ if(r.floorcnt){
+ row << r.floorcnt
+ }else{
+ row << ""
+ }
+
+ if(r.houseaspect && !r.houseaspect.contains("暂")){
+ row << r.houseaspect
+ }else{
+ row << ""
+ }
+
+ row << "房价网"
+ rows << row
+ }
+
+ println d['R'] +" "+results.size()
+ }
+ }
+ }
+ ExcelUtils.write(output, rows, "评估案例")
+ }
+
+
+ def createDealPrice(path, output){
+ def datas = ExcelUtils.read(path, 0, 1)
+ def rows = []
+ rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
+ def keys = []
+ for(def d : datas) {
+ def commid= d['R']
+ println "[createDealPrice] commid:"+commid
+ if (commid) {
+ if (keys.contains(commid)) {
+ continue
+ }
+ keys << commid
+ def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20)
+ if(jsonObject && jsonObject.result) {
+ def results = jsonObject.result
+ for (def r : results) {
+ def row = []
+ row << "成交"
+ row << r.districtName
+ if(r.type){
+ row << r.type
+ }else{
+ row << ""
+ }
+
+ if(r.room){
+ def roomStr = r.room+"室"
+ if(r.hall){
+ roomStr = roomStr +r.hall+"厅"
+ }
+ row << roomStr
+ }else{
+ row << ""
+ }
+ row << r.area
+ row << r.fitment
+ row << r.tradeDate.substring(0, r.tradeDate.indexOf("T"))
+ row << ""
+ row << r.avgPrice
+ row << r.totalPrice
+ if(r.floor){
+ row << r.floor
+ }else{
+ row << ""
+ }
+ if(r.height){
+ row << r.height
+ }else{
+ row << ""
+ }
+
+ if(r.direction && !r.direction.contains("暂")){
+ row << r.direction
+ }else{
+ row << ""
+ }
+
+ row << r.source
+ rows << row
+ }
+
+ println d['R'] +" "+results.size()
+ }
+ }
+ }
+ ExcelUtils.write(output, rows, "成交案例")
+ }
+
+ def addAuthor(def path){
+ def authors = ["丁世菁","王文艳","董彦均","曹玉彦","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","穆良书","沈燕芳"]
+ def personTotal = authors.size()
+ def avg = Math.ceil(total/personTotal)
+
+ def index = 0
+ def count = 0;
+ ExcelUtils.modify(path, { row ->
+ if(row['B']){
+ if(index < avg){
+ row['X'] = authors[count]
+ }
+ index++
+ if(index >= avg){
+ count++
+ index = 0
+ }
+ }
+ });
+ }
+
+ def findAvgPrice(def commid){
+ def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
+ def avg_price = null
+ sqlInstance.eachRow(sql, { r ->
+ avg_price = r.target_avg_price
+ });
+ return avg_price
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
index d6ab63366..b8e3cd5a1 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy
@@ -68,6 +68,10 @@ class CommonDatautils {
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
+ public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
+ public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
+
+
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -586,8 +590,8 @@ class CommonDatautils {
}
- public static def findDistrict(String city, String region, String name, String cat){
-
+ public static def findDistrictCount(String city, String region, String name, String cat){
+
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
@@ -598,7 +602,24 @@ class CommonDatautils {
q.put("cat", cat)
q.put("status", 0)
-
+ def count = getDataStoreConnection().count(q);
+ return count;
+ }
+
+
+ public static def findDistrict(String city, String region, String name, String cat){
+
+ DBObject q = new BasicDBObject();
+ q.put("city", city);
+ if(region){
+ q.put("region", region);
+ }
+
+ q.put("name", name);
+ q.put("cat", cat)
+ q.put("status", 0)
+
+
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
@@ -609,6 +630,7 @@ class CommonDatautils {
}
+
public static def findDistrictList(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
@@ -704,7 +726,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
-// q.put("name","吴中路9弄小区")
+// q.put("name","森望芳草绿岸")
DBObject keys = new BasicDBObject();
if(fields){
@@ -827,16 +849,24 @@ class CommonDatautils {
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
+ keys.put("lat2", 1);
+ keys.put("lng2", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
keys.put("sid", 1)
- DSCursor cursor = getDataStoreConnection().find(q, keys);
- if(cursor){
- for(def obj : cursor){
- return obj;
+ try{
+ DSCursor cursor = getDataStoreConnection().find(q, keys);
+ if(cursor){
+ for(def obj : cursor){
+ return obj;
+ }
}
+ }catch(Exception e){
+ println "error_id:"+id
+ e.printStackTrace()
}
+
return null;
}
@@ -1053,9 +1083,16 @@ class CommonDatautils {
public static def queryDistrictById(def id){
+ def dbo = null
DBObject q = new BasicDBObject();
- q.put("_id", new ObjectId(id));
- def dbo = getDataStoreConnection().findOne(q);
+ try{
+ q.put("_id", new ObjectId(id.trim().toString()));
+ dbo = getDataStoreConnection().findOne(q);
+ }catch(Exception e){
+ println id
+ }
+
+
return dbo;
}
@@ -2385,6 +2422,7 @@ class CommonDatautils {
q.put("block", block);
}
q.put("cat", cat);
+ q.put("status",0)
def cur = getDataStoreConnection().find(q);
return cur.count();
}
@@ -3041,18 +3079,26 @@ class CommonDatautils {
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate));
- DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)).limit(1)
+ DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
DBObject dbo;
+ def prices = []
+ def index = 0
while(cur.hasNext()) {
- dbo = cur.next()
- def price = dbo.get("avg_price");
- if(price){
- break;
+ def obj = cur.next()
+ if(index == 0){
+ dbo = obj
}
+ prices << obj.get("avg_price");
+ index++
}
- return dbo;
+ if(prices){
+ def avg_price = (int)(prices.sum()/prices.size())
+ dbo.avg_price = avg_price
+ return dbo;
+ }
+ return null;
}
@@ -3184,5 +3230,19 @@ class CommonDatautils {
def avg_radios = (int)(radios.sum()/radios.size())
return avg_radios;
}
+
+ static def findLoudong(commid){
+ def q = new BasicDBObject()
+ q.put("id_in_seaweed", commid)
+ def count = LOUDONG_LINE_COLL.count(q)
+ return count;
+ }
+
+ static def findHouseLoudong(commid){
+ def q = new BasicDBObject()
+ q.put("district_id", commid)
+ def count = HOUSE_LOUDONG_COLL.count(q)
+ return count;
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy
index 2a2f5890f..63d7f841f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/DistrictAroundFacilitiesBatch.groovy
@@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch {
@Override
public void process(Object args) {
- def cities = ["北京", "上海"]
+ def cities = ["北京", "上海"]//
for (def city : cities) {
def ret = CommonDatautils.getAllDistricts("district", city)
// def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district");
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy
index e8cd3476a..bb7b11f46 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy
@@ -6,10 +6,16 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jusfoun.JusfounCommon
+import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
+import org.bson.types.ObjectId
+import java.text.DecimalFormat
import java.util.concurrent.TimeUnit
/**
@@ -19,8 +25,11 @@ import java.util.concurrent.TimeUnit
class DistrictPriceExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
+ def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def charset = "GBK"
+ public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
+
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -32,6 +41,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createDatas("福州")
+ saveToDB(ret)
+
+
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){
@@ -76,10 +88,10 @@ class DistrictPriceExportBatch extends BaseBatch{
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
-
- def lastDate = new Date()
- def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
-
+ def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
+ def dataStr = DateUtils.add(endDateStr, -30)
+ def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
+ def lastDate = DateUtils.parseFromTightByPattern(endDateStr, "yyyy-MM-dd")
// List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv"))
// def map2 = [:]
// for(def s : list2) {
@@ -88,17 +100,34 @@ class DistrictPriceExportBatch extends BaseBatch{
// map2.put(key, row[5])
// }
+ def lastMonthPriceMap = findFzDistrictStat("福州",lastMonthDate)
def result = []
for(def rMap : districts){
+
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
if(dbo){
+ def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
+ def curPrice = dbo.get("avg_price")
+ if(lastMonthPrice && curPrice){
+ def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
+ if(Math.abs(markup)>3){
+ def randMarkup = nextDouble(0, 3)
+ if(markup < 0){
+ markup = randMarkup*(-1)
+ }else{
+ markup = randMarkup
+ }
+ curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
+ }
+ }
+
def m = []
m << dbo.get("_id").toString()//均价ID
m << rMap.get("_id").toString()//楼盘ID
m << rMap.city
m << rMap.region
m << rMap.name
- m << dbo.get("avg_price")
+ m << curPrice
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
// if(lastMonthPrice && dbo.get("avg_price")){
// lastMonthPrice = lastMonthPrice.toInteger()
@@ -125,6 +154,19 @@ class DistrictPriceExportBatch extends BaseBatch{
return result
}
+
+ def findFzDistrictStat(city,lastMonthDate){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("month", lastMonthDate)
+ def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
+ def priceMap = [:]
+ for(def d : dbcursor){
+ priceMap.put(d.comm_id, d.avg_price)
+ }
+ return priceMap
+ }
+
def export(def fileName, def datas){
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
writeCSV(headers, fileName, datas);
@@ -152,5 +194,44 @@ class DistrictPriceExportBatch extends BaseBatch{
file.append(System.getProperty("line.separator"));
}
}
+
+ //随机浮点数
+ public static double nextDouble(final double min, final double max) throws Exception {
+
+ DecimalFormat df = new DecimalFormat("#.00");
+ if (max < min) {
+ throw new Exception("min < max");
+ }
+ if (min == max) {
+ return min;
+ }
+ return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
+ }
+
+ def saveToDB(rets){
+ for(def r : rets){
+ def q = new BasicDBObject()
+ q.put("month", curDate.toInteger())
+ q.put("comm_id", r[1])
+ def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
+ if(obj){
+ def udata = new BasicDBObject()
+ udata.put("m_date", new Date())
+ udata.put("avg_price", r[5])
+ FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
+ }else{
+ def udata = new BasicDBObject()
+ def o = new BasicDBObject()
+ o.put("_id", new ObjectId(r[0]))
+ o.put("comm_id", r[1])
+ o.put("city", r[2])
+ o.put("region", r[3])
+ o.put("name", r[4])
+ o.put("avg_price", r[5].toInteger())
+ o.put("month", r[6].toInteger())
+ FZ_DISTRICT_STAT_COLL.save(o)
+ }
+ }
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy
index ba848d886..81c5624d1 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy
@@ -8,7 +8,9 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
+import com.rabbitmq.client.impl.AMQImpl
import org.apache.commons.io.FileUtils
+import org.bson.types.ObjectId
/**
* 福州数据结果统计
@@ -17,16 +19,42 @@ class FzDataResultStat {
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
+ public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
+
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
task.regionPriceTrendRun()
+//// task.margeExcel()
+// task.saveCsvData()
+ }
+
+ def margeExcel(){
+
+ def list2= ExcelUtils.read("D:\\数据\\福州\\价格比对_202204_新2.xlsx", 0 ,1)
+
+ def map2 = [:]
+ for(def s : list2){
+ def key = s["A"]+"##"+s["B"]+"##"+s["C"]
+ map2.put(key, s["D"])
+ }
+
+ ExcelUtils.multiThreadingModify("D:\\数据\\福州\\价格比对_202204_新3.xlsx",{row->
+ if(row["A"]){
+ def key = row["A"]+"##"+row["B"]+"##"+row["C"]
+ def price = map2.get(key)
+ if(price){
+ row['L'] = price
+ }
+ }
+ })
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
+
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
@@ -137,16 +165,13 @@ class FzDataResultStat {
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
- def price = 0;
+ def prices = [];
while(cur.hasNext()) {
DBObject dbo = cur.next()
- price = dbo.get("avg_price");
- if(price){
- break;
- }
+ prices << dbo.get("avg_price");
}
-
- return price;
+ def avg_price = (int)(prices.sum()/prices.size())
+ return avg_price;
}
@@ -165,4 +190,45 @@ class FzDataResultStat {
weekends = weekends.reverse();
return weekends;
}
+
+
+ def saveCsvData(){
+ def files = getAllFiles("C:\\Users\\gaojuan\\福州数据(1)")
+ if(files){
+ for(def filepath : files){
+ List list1= FileUtils.readLines(filepath)
+ println filepath
+ for(def s : list1){
+ def row = s.split("@\\|@")
+ def o = new BasicDBObject()
+ o.put("_id", new ObjectId(row[0]))
+ o.put("comm_id", row[1])
+ o.put("city", row[2])
+ o.put("region", row[3])
+ o.put("name", row[4])
+ o.put("avg_price", row[5].toInteger())
+ o.put("month", row[6].toInteger())
+ FZ_DISTRICT_STAT_COLL.save(o)
+
+ }
+ }
+ }
+ }
+
+ def static getAllFiles(def path) {
+
+ File file = new File(path);
+ File[] tempList = file.listFiles();
+
+ def files = [];
+ for (int i = 0; i < tempList.length; i++) {
+ if (tempList[i].isFile()) {
+ files << tempList[i];
+ }
+ if (tempList[i].isDirectory()) {
+ System.out.println("文件夹:" + tempList[i]);
+ }
+ }
+ return files;
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
index 5a6b21aac..7e5074c7c 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
-def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv")
+def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv")
def count0 = 0
def count1 = 0
def count2 = 0
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy
index 244497f17..cd6252fb6 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/TradeExportBatch.groovy
@@ -112,22 +112,28 @@ class TradeExportBatch extends BaseBatch{
// q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate))
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000);
def rets = []
+ def keys = []
if (cursor != null) {
while (cursor.hasNext()) {
DBObject dbo = cursor.next();
def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name);
if(district){
- def ret = [];
- ret << dbo._id.toString()
- ret << district._id.toString()
- ret << dbo.city
- ret << dbo.region
- ret << dbo.district_name
- ret << dbo.avg_price
- ret << dbo.area
- ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
- ret << dbo.total_price
- rets << ret;
+ def tradeDateStr = DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
+ def key = district._id.toString() +"##" + dbo.avg_price +"##"+dbo.area+"##"+tradeDateStr
+ if(!keys.contains(key)){
+ keys << key
+ def ret = [];
+ ret << dbo._id.toString()
+ ret << district._id.toString()
+ ret << dbo.city
+ ret << dbo.region
+ ret << dbo.district_name
+ ret << dbo.avg_price
+ ret << dbo.area
+ ret << tradeDateStr
+ ret << dbo.total_price
+ rets << ret;
+ }
}
}
println "成交总计:"+rets.size()
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy
index 4628a207b..d5123668b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHousePriceStat.groovy
@@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{
def run(){
def cityMap = CommonDatautils.getAllCityRegions()
cityMap.each{city,regions->
- findPriceByCity(city, regions);
+
+ findPriceByCity(city, regions);
}
}
@@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{
q.put("region" , r.region)
q.put("month", r.month)
def mom = null
- if(lastObj){
- mom = (r.price -lastObj.avg_price)/lastObj.avg_price
- mom = DataHandleUtils.getDoubleRound(4, mom)
+ if(lastObj && lastObj.avg_price && r.price){
+ mom = (r.price -lastObj.avg_price)/lastObj.avg_price
+ mom = DataHandleUtils.getDoubleRound(4, mom)
}
if(obj){
if(obj.avg_price != r.price){
@@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{
def findEsAvgPrice(city, region){
def detail = [:]
- detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1])
+ detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0])
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
def must = []
@@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{
if(!priceValue){
priceValue = row['1-metric']
}
- if(priceValue){
+ if(priceValue && priceValue.value){
price = priceValue.value.toInteger()
}
result.price = price;
def date = row.key_as_string.replace("-","").substring(0, 6)
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
if(date!=curMonth){
- result.month = date.toInteger()
- rets << result
+ if(date.toInteger()>200001 && result.price){
+ result.month = date.toInteger()
+ rets << result
+ }
+
}
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
index 77239e3ea..e4b621c99 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy
@@ -164,6 +164,7 @@ class OpenApiUtils {
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null;
}
+ address = address.trim().replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
List poiList = null;
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy
index c60a811b5..19709dd22 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCityPriceStat.groovy
@@ -18,44 +18,62 @@ class CcbCityPriceStat {
task.run()
}
def run(){
- def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
+ def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
def rets = []
+ def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")
+ def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")
+
for(def city : citys){
- def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
- def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
- def cityRets = []
- for(def dist : districts){
- def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
- def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
- if(tradeAvgPrice && weekendPrice){
- def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
- cityRets << row
- println row.toString()
+ def matchTotalCount = findMatchTradeCount(city, startDate, endDate)
+ println city +" "+matchTotalCount
+ if(matchTotalCount){
+ def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
+ def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29"))
+ def cityRets = []
+ for(def dist : districts){
+ def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01")
+ def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
+ if(tradeAvgPrice && weekendPrice){
+ def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
+ cityRets << row
+ println row.toString()
+ }
}
+ print city +" "+ cityRets.size()
+ rets.addAll(cityRets)
}
- print city +" "+ cityRets.size()
- rets.addAll(cityRets)
+
}
println "总量:"+rets.size()
- ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
+ ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有")
+ }
+
+
+ def findMatchTradeCount(city, startDate, endDate){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
+ q.put("match_id", new BasicDBObject('$exists', true))
+ def count = DEAL_PRICE_COLL.count(q)
+ return count
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
- q.put("city", dist.city)
- q.put("region", dist.region)
-
- if(dist.alias){
- if(!dist.alias.contains(dist.name)){
- dist.alias << dist.name
- }
- q.put("district_name", new BasicDBObject('$in', dist.alias))
- }else{
- q.put("district_name", dist.name)
- }
-
+// q.put("city", dist.city)
+// q.put("region", dist.region)
+//
+// if(dist.alias){
+// if(!dist.alias.contains(dist.name)){
+// dist.alias << dist.name
+// }
+// q.put("district_name", new BasicDBObject('$in', dist.alias))
+// }else{
+// q.put("district_name", dist.name)
+// }
+ q.put("match_id", dist._id.toString())
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy
index 1dabf3ada..66ff6a8f3 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy
@@ -199,18 +199,18 @@ class CcbCodeUtils {
static def getBuildStructureCode(def buildStructure){
if(buildStructure){
if(buildStructure.contains("钢结构")){
- return "001";
+ return "3";
}else if(buildStructure.contains("钢、钢筋混凝土结构")){
- return "002";
+ return "1";
}else if(buildStructure.contains("钢筋混凝土结构")){
- return "003";
+ return "1";
}else if(buildStructure.contains("砖木结构")){
- return "004";
+ return "10";
}else if(buildStructure.contains("混合结构")){
- return "005";
+ return "2";
}
}
- return "006";
+ return "11";
}
static def getDistHouseTypeCode(def estateType){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy
index 3e4981856..23513b80b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy
@@ -50,4 +50,32 @@ class FileUtils {
}
out.close();
}
+
+ static def createTxtFile(ret, fileName, charset, isAppend){
+ File file = new File(fileName);
+ if (!file.getParentFile().exists()) {
+ boolean result = file.getParentFile().mkdirs();
+ if (!result) {
+ System.out.println("创建目录失败");
+ }
+ }
+
+ OutputStream out = new FileOutputStream(fileName, isAppend);
+ def sb = new StringBuffer();
+ out.write(sb.toString().getBytes(charset));
+ sb.setLength(0)
+ def index = 0
+ for(def row : ret){
+ if(row){
+ index++
+ sb.append(row)
+ sb.append("\n");
+ }
+ }
+ if(sb.length()>0){
+ out.write(sb.toString().getBytes(charset));
+ sb.setLength(0)
+ }
+ out.close();
+ }
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy
new file mode 100644
index 000000000..d55350598
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeAppCrawl.groovy
@@ -0,0 +1,257 @@
+package com.fangjia.script.gj.crawl
+
+import com.fangjia.script.crawl.JQueryUtils
+import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
+import com.fangjia.script.thirdPriceCrawl.Utils
+import com.fangjia.script.util.ABuYunUtil
+import com.fangjia.script.util.MongoUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.rabbitmq.client.impl.AMQImpl
+import net.sf.json.JSONObject
+import org.jsoup.Jsoup
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.chrome.ChromeDriver
+
+class AnjukeAppCrawl {
+
+ public static final SHORT_DATE = Utils.getLastMonth();
+ def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
+ public static def SOURCE = "安居客";
+ DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
+
+ static main(args) {
+ def task = new AnjukeAppCrawl();
+// task.run();
+// task.run2()
+
+ task.addThirdPrice()
+ }
+
+ def run2(){
+ def url = "https://m.anjuke.com/sh/community/"
+ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
+ System.setProperty("http.nonProxyHosts", "localhost");
+ WebDriver driver = new ChromeDriver()
+ def links = getRegion(url, driver)
+ for(def link : links){
+ def doc = getDocument(url, driver)
+ def str = doc.toString().substring(doc.toString().indexOf(""))
+ if(str){
+ str = str.substring(str.indexOf("} ("), str.indexOf("));"))
+ def arrs = str.split(",")
+ pritln arrs.size()
+ }
+
+ }
+ }
+
+ def getDocument(url, driver){
+ def stop = false
+ def error = 0
+ def doc = null
+ while(!stop){
+ try{
+ driver.get(url)
+ doc = Jsoup.parse(driver.getPageSource());
+ if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("登录")){
+ stop = true;
+ }else{
+ error++
+ driver.manage().deleteAllCookies()
+ println "[${error}] error: "+url
+ Thread.sleep(5000)
+ }
+ }catch(Exception e){
+ error++
+ }
+ if(error>=20){
+ stop = true;
+ }
+ }
+ return doc
+ }
+
+ def getRegion(url, driver){
+ //def doc = getDocument(url, driver)
+ def header = [:]
+ header.put("accept","application/json, text/plain, */*")
+ header.put("accept-encoding","gzip, deflate, br")
+ header.put("accept-language","zh-CN,zh;q=0.9")
+ header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
+ header.put("referer","https://m.anjuke.com/fz/")
+ header.put("sec-fetch-dest","empty")
+ header.put("sec-fetch-mode","cors")
+ header.put("sec-fetch-site","same-origin")
+ header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
+ header.put("sw8-correlation","c291cmNl:YmVpZG91")
+ header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
+
+ def doc = ABuYunUtil.doGet(url, null, header, "utf-8")
+ def links = []
+ if(doc){
+ def eles = JQueryUtils.select(".column2 a", doc)
+ for(def e : eles){
+ def href = e.attr("href")
+ links << href
+ }
+ }
+ return links;
+ }
+
+
+
+
+ def run(){
+ //上海:11,北京:14
+ //"仓山":4071,
+ // def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
+ // def regionMap = ["浦东":7,"闵行":11, "松江":15,"宝山":13,"嘉定":14,"徐汇":5,"青浦":19,"静安":2,"普陀":10,"杨浦":9,"奉贤":17,"黄浦":4,"虹口":8,"长宁":6,"金山":18,"崇明":20]
+ def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678]
+ def cityMap = ["北京":regionMap]
+ def cityId = 14
+ regionMap.each{region, area_id->
+ for(def i=1;i<40;i++){
+ def stop = false;
+ def all_stop = false
+ def error_count=0
+ while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
+ //城口
+ def flagMap = getInfo(stop, i, area_id, cityId, "上海")
+ stop = flagMap.stop
+ all_stop = flagMap.all_stop
+ error_count++
+ if(error_count>=20){
+ stop = true
+ all_stop = true
+ }
+ }
+ if(all_stop){
+ break;
+ }
+ }
+ }
+ }
+
+ def getInfo(def stop, def i, def area_id, def cityId, def city){
+
+ def map = [:]
+ def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
+
+ def header = [:]
+ header.put("accept","application/json, text/plain, */*")
+ header.put("accept-encoding","gzip, deflate, br")
+ header.put("accept-language","zh-CN,zh;q=0.9")
+ header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
+ header.put("referer","https://m.anjuke.com/sh/community/")
+ header.put("sec-fetch-dest","empty")
+ header.put("sec-fetch-mode","cors")
+ header.put("sec-fetch-site","same-origin")
+ header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
+ header.put("sw8-correlation","c291cmNl:YmVpZG91")
+ header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
+
+ def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
+ def all_stop = false;
+ def error_count = 0
+ if (ret) {
+ JSONObject jsonObj = JSONObject.fromObject(ret);
+ if(jsonObj && jsonObj.status=="ok" ){
+ def data = jsonObj.data
+ if(data && data.total.toInteger()>0){
+ def communities = data.communities
+
+ println "【${i}】 "+communities.size() +" "+url
+ if(communities.size()>0){
+ save(city, communities)
+ Thread.sleep(10000)
+ stop = true;
+ }else{
+ stop = true;
+ println url +" 返回0"
+ all_stop = true;
+ }
+
+ }else{
+ stop = false;
+ error_count ++
+ println url +" 无数据"
+ }
+
+
+ }else{
+ stop = false;
+ Thread.sleep(5000)
+ }
+ }
+
+ map.stop = stop
+ map.all_stop = all_stop;
+ return map;
+ }
+
+ def save(city, def ret){
+ def index = 0
+ for(def o : ret){
+ index++
+ def obj = findIsExist(o.jumpAction)
+ if(!obj){
+ def udate = new BasicDBObject()
+ o.each{k,v->
+ udate.put(k,v)
+ }
+ udate.put("city", city)
+ udate.put("c_date", new Date())
+ detail.save(udate)
+ println index+" [保存]"+o.jumpAction
+ }else{
+ println index+" [已存在]"+o.jumpAction
+ }
+
+ }
+ }
+
+ def findIsExist(def url){
+ def q = new BasicDBObject()
+ q.put("jumpAction",url)
+ return detail.findOne(q);
+ }
+
+ def addThirdPrice(){
+ def q = new BasicDBObject()
+ def dbcursor = detail.find(q)
+ def index = 0
+ for(def d : dbcursor){
+ index++
+ def city = d.base.cityName
+ def region = d.base.areaName
+ def name = d.base.name
+ def obj = findByParams(city, region, name);
+ if(!obj){
+ def data = new BasicDBObject()
+ data.put("district_id",d.base.id)
+ data.put("city", city)
+ data.put("region", region )
+ data.put("name", name)
+ data.put("address",d.base.address)
+ data.put("avg_price", d.priceInfo.price)
+ data.put("source", SOURCE)
+ data.put("s_date", SHORT_DATE)
+ data.put("c_date", d.c_date)
+ thirdPartyColl.save(data)
+ println index +" "+data
+ }
+ }
+ }
+
+
+ def findByParams(city, region, name){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("region", region)
+ q.put("name", name)
+ q.put("source", SOURCE)
+ q.put("s_date", SHORT_DATE)
+ return thirdPartyColl.findOne(q);
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy
index d4f3001b3..ab7aa4278 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDetailCrawl.groovy
@@ -11,9 +11,15 @@ import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
+import org.openqa.selenium.remote.DesiredCapabilities
+import org.openqa.selenium.remote.RemoteWebDriver
class AnjukeDetailCrawl {
+
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
+
+ DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url")
+
static main(args) {
def task = new AnjukeDetailCrawl();
task.run();
@@ -21,9 +27,15 @@ class AnjukeDetailCrawl {
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
- def url = "https://chongqing.anjuke.com/community/?from=navigation"
+ def url = "https://chongqing.anjuke.com/"
WebDriver driver = new ChromeDriver()
+// DesiredCapabilities capabilities = DesiredCapabilities.chrome();
+// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL
+// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities);
+
+ driver.manage().deleteAllCookies()
def regions = getRegions(driver, url)
+ println "区域链接: "+regions.size()
getAllRegions(driver, regions)
}
@@ -31,44 +43,88 @@ class AnjukeDetailCrawl {
def getAllRegions(driver, regions){
for(def url : regions){
- driver.get(url)
- def doc = Jsoup.parse(driver.getPageSource());
+ def doc = getDocument(url, driver)
def total = getTotal(doc)
if(total) {
if (total > 1250) {
-
+ def links = getMoreLinks(url, driver)
+ println url +" "+links.size()
+ saveDetailLinks(links)
+// parse(links,driver)
} else {
def links = getAllLinks(url, driver)
- parse(links,driver)
+ println url +" "+links.size()
+ saveDetailLinks(links)
+// parse(links,driver)
}
}
}
}
+ def saveDetailLinks(links){
+ for(def url : links){
+ def data = new BasicDBObject()
+ data.put("city","重庆")
+ data.put("url", url)
+ data.put("source", "安居客")
+ data.put("c_date", new Date())
+ urlColl.save(data)
+ }
+ }
+
+ def getMoreLinks(url, driver){
+ def doc = Jsoup.parse(driver.getPageSource());
+ def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc)
+ def links = []
+ if(pricelinks){
+ for(def link : pricelinks){
+ if(link.text()!="不限"){
+ def urlLink = link.attr("href")
+ getDocument(urlLink, driver)
+// driver.get(urlLink)
+ links.addAll(getAllLinks(urlLink, driver))
+ println urlLink +" "+links.size()
+ }
+ }
+ }
+ return links;
+ }
+
def getAllLinks(url, driver){
def stop = false
def urls = []
- while(!stop) {
- def doc = Jsoup.parse(driver.getPageSource());
+ def doc = Jsoup.parse(driver.getPageSource());
+ def total = getTotal(doc)
+ def page = getPage(total, 25)
+ def pages = []
+ for(def i=1;i<=page;i++){
+ pages << url.substring(0, url.size()-1)+"-p"+i+"/"
+ }
+ for(def pageUrl : pages){
+ doc = getDocument(pageUrl, driver)
def details = JQueryUtils.select(".list-cell a.li-row", doc)
-
for (def d : details) {
- urls << d.attr("href")
- }
- def nextEle = JQueryUtils.select(".next", doc)
- if (nextEle) {
- driver.findElement(By.cssSelector(".next")).click()
- }else{
- stop = true
- }
- if(urls.size()>20){
- stop = true
+ def link = d.attr("href")
+ if(!urls.contains(link)){
+ urls << d.attr("href")
+ }
}
}
return urls
}
+ def getPage(def total, def pageSize){
+ if(total && total.toInteger()>0){
+ def page = total.toInteger()/pageSize;
+ if(page>(int)page){
+ page = (int)page + 1
+ return page
+ }
+ return (int)page;
+ }
+ return null
+ }
def getTotal(doc){
def totalText = JQueryUtils.selectText(".total-info", doc)
@@ -82,9 +138,19 @@ class AnjukeDetailCrawl {
}
def getRegions(driver, url){
+// def doc = getDocument(url, driver)
driver.get(url)
- Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource());
+ def els = JQueryUtils.select("a:contains(小区)", doc)
+ if(els){
+ def href = els[0].attr("href")
+ doc = Jsoup.parse(driver.getPageSource());
+ // driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click()
+// driver.findElement(By.linkText("小区")).click()
+ doc = getDocument(href, driver)
+ }
+
+
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
// if(loginIfm){
// def href = loginIfm[0].attr("src")
@@ -107,23 +173,35 @@ class AnjukeDetailCrawl {
def regionEles = JQueryUtils.select(".region-parents li a", doc)
def urls = []
for(def e : regionEles){
- urls << e.attr("href")
+ if(e.text()!="不限"){
+ urls << e.attr("href")
+ }
}
return urls;
}
+ def getNontCheckPage(url, driver, doc){
+ def stop = false;
+ while(!stop){
+ if(doc){
+
+ }
+ }
+ return doc;
+ }
+
def parse(def links, driver){
def rets = []
for(def link : links){
def district = parsing(link, driver)
if(district){
- rets << district
+ // rets << district
- if (rets.size() % 100 == 0) {
- def add= saveData(rets)
- println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
- rets = []
- }
+ // if (rets.size() % 100 == 0) {
+ saveObj(district)
+ println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
+ // rets = []
+ // }
}
}
if (rets.size()> 0) {
@@ -136,33 +214,61 @@ class AnjukeDetailCrawl {
def saveData(def results){
def add = 0
for(def ret : results){
- def q = new BasicDBObject()
- q.put("url", ret.url)
- def dbo = detailColl.findOne(q);
-
- DBObject obj = new BasicDBObject();
- ret.each{k,v->
- obj.put(k,v)
- }
-
- if(dbo){
- obj.m_date = new Date()
- detailColl.update(q, new BasicDBObject('$set', obj))
- }else{
- add++
- obj.c_date = new Date()
- obj.m_date = new Date()
- try{
- detailColl.save(obj)
- }catch(Exception e){
- println obj.url
- }
- }
+ saveObj(ret)
}
- return add
}
+ def saveObj(def ret){
+ def q = new BasicDBObject()
+ q.put("url", ret.url)
+ def dbo = detailColl.findOne(q);
+
+ DBObject obj = new BasicDBObject();
+ ret.each{k,v->
+ obj.put(k,v)
+ }
+
+ if(dbo){
+ obj.m_date = new Date()
+ detailColl.update(q, new BasicDBObject('$set', obj))
+ }else{
+
+ obj.c_date = new Date()
+ obj.m_date = new Date()
+ try{
+ detailColl.save(obj)
+ }catch(Exception e){
+ println obj.url
+ }
+ }
+ }
+ def getDocument(url, driver){
+ def stop = false
+ def error = 0
+ def doc = null
+ while(!stop){
+ try{
+ driver.get(url)
+ doc = Jsoup.parse(driver.getPageSource());
+ if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("登录")){
+ stop = true;
+ }else{
+ error++
+ driver.manage().deleteAllCookies()
+ println "[${error}] error: "+url
+ Thread.sleep(5000)
+ }
+ }catch(Exception e){
+ error++
+ }
+ if(error>=20){
+ stop = true;
+ }
+ }
+ return doc
+ }
+
def parsing(url, driver){
@@ -191,10 +297,9 @@ class AnjukeDetailCrawl {
def infoDiv = JQueryUtils.select(".info-list", doc)
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
- district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
- district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
- district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
-
+ district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text())
+ district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
+ district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
if (houseCountStr) {
houseCountStr = houseCountStr.replace("户", "")
@@ -202,9 +307,13 @@ class AnjukeDetailCrawl {
district.house_count = houseCountStr.toInteger()
}
}
-
- district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
+ district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
+ district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text())
+
+ district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
+ district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text())
+
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
if(carSiteStr){
if(carSiteStr.contains("(")){
@@ -214,16 +323,19 @@ class AnjukeDetailCrawl {
district.car_site = carSiteStr
}
}
- district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
- district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text())
- district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
- district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
- district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
+ district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text()
+ district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
+
+ district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
+ district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text())
+
+ district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
+ district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
district.url = url
district.c_date = new Date()
district.m_date = new Date()
+ district.crawl_type = "chrome"
district.root = "anjuke.com"
-
stop = true;
}else{
Thread.sleep(3000)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy
new file mode 100644
index 000000000..f990711be
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeDistrictMatch.groovy
@@ -0,0 +1,89 @@
+package com.fangjia.script.gj.crawl
+
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.temp.shenjian.util.SeaweedUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.OpenApiClient
+
+class AnjukeDistrictMatch {
+
+ static main(args) {
+ def task = new AnjukeDistrictMatch();
+ task.run();
+// task.run1()
+ }
+
+ def run1(){
+ def regionPointsMap = findRegionRange("重庆")
+ def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
+ }
+ def run(){
+
+
+
+ def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
+ ExcelUtils.multiThreadingModify(filePath,{row->
+ if(row['A'] && !row['J']){
+ def city = row['B']
+ def region = row['C']
+ def name = row['D']
+ def address = row['E']
+// def flag = "别名匹配"
+
+
+// def dist = matchDist(city, null, name)
+// if(!row['J'] && name.contains("(")){
+// name = name.replace("(${region})","").replace("(","").replace(")","")
+// flag = "去括号匹配"
+// dist = matchDist(city, region, name)
+// }
+// if(!dist){
+// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
+// flag = "映射别名匹配"
+// }
+// if(!dist) {
+// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
+// if(rets && rets.size()==1){
+// dist = rets[0]
+// }
+// }
+
+//
+// if(!dist){
+// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district")
+// flag = "地址匹配"
+// if(dists.size()>1){
+// println address +" 地址相同:"+dists.size()
+// }else{
+// if(dists.size()==1){
+// dist = dists[0]
+// }
+// }
+// }
+
+// if(dist){
+// row['J'] = flag
+// row['K'] = dist._id.toString()
+// row['L'] = dist.region
+// row['M'] = dist.name
+// row['N'] = dist.address
+// }
+
+ def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
+ if(jsonObject != null && jsonObject.precise!=-1) {
+// row['J'] = "匹配接口"
+// row['K'] = jsonObject.get("id")
+// row['L'] = jsonObject.get("district")
+// row['M'] = jsonObject.get("name")
+// row['N'] = jsonObject.get("address")
+ row['M'] = jsonObject.get("name")
+ }
+ }
+ })
+ }
+
+ def matchDist(city, region, name){
+ def dist = CommonDatautils.findDistrict(city, region, name, "district")
+ return dist;
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy
new file mode 100644
index 000000000..1773bdd8f
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeHouseCrawl.groovy
@@ -0,0 +1,189 @@
+package com.fangjia.script.gj.crawl
+
+import com.alibaba.fastjson.JSON
+import com.alibaba.fastjson.JSONObject
+import com.fangjia.core.RegionData
+import com.fangjia.script.crawl.JQueryUtils
+import com.fangjia.script.gj.EsUtils
+import com.fangjia.script.temp.shenjian.util.MessageUtils
+import com.fangjia.script.util.CommonUtils
+import org.jsoup.Jsoup
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.chrome.ChromeDriver
+
+class AnjukeHouseCrawl {
+ def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
+ static main(args) {
+ def task = new AnjukeHouseCrawl();
+ task.run();
+ }
+
+ def run(){
+ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
+ System.setProperty("http.nonProxyHosts", "localhost");
+ WebDriver driver = new ChromeDriver()
+ for(def i=24;i<50;i++){
+ def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
+ def doc = getDocument(url, driver)
+ def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
+ println "页数:"+i+" 记录数:"+eles.size()
+ if(eles){
+ parser(eles, driver)
+ }
+ }
+ }
+
+ def parser(eles, driver){
+ def index = 0
+ for(def e : eles){
+ index ++
+ def href = e.attr("href")
+ def doc = getDocument(href, driver)
+// parserHtml(doc)
+ def newUrl = href.substring(0, href.indexOf("?"))
+ sendMq(newUrl, doc)
+
+ println "[ "+index +"] "+ newUrl
+ }
+ }
+
+ def sendMq(href, doc){
+ JSON json = new JSONObject()
+ json.put("project", "sell_anjuke")
+ json.put("url", href)
+
+ JSON result = new JSONObject()
+ result.put("content", doc.toString())
+ json.put("result", result)
+
+ byte[] packedMsg = MessageUtils.pack(json)
+ CommonUtils.sendMessage("result2parse", packedMsg)
+ }
+
+ def parserHtml(doc) {
+ String city = null
+ def result = [:]
+ result.siteCode = "安居客"
+ if (doc) {
+ result.title = JQueryUtils.selectText([".banner-title h1"], doc)
+ def heads = JQueryUtils.select(".crumbs a", doc)
+
+ if (heads && heads.size() >= 5) {
+ result.city = heads[0].text().replace("房产网", "")
+ def orgRegion = heads[2].text().replace("二手房", "")
+ result.region = RegionData.formatRegion(result.city, orgRegion.trim())
+ result.block = heads[3].text().replace("二手房", "")
+ result.district_name = heads[4].text()
+ } else if (heads && heads.size() >= 3) {
+ result.city = heads[0].text().replace("房产网", "")
+ result.district_name = heads[2].text()
+ def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
+ if (eles && eles.size() > 0) {
+ def orgRegion = eles[0].text()
+ result.region = RegionData.formatRegion(result.city, orgRegion.trim())
+ }
+ }
+
+ if (!result.district_name) {
+ println " [未解析出(城市-区域-名称)信息]"
+ }
+ println result
+
+
+ def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
+ if (priceStr) {
+ if (priceStr.isNumber()) {
+ result.total_price = priceStr.toDouble() * 10000
+ }
+ }
+ def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
+ if (roomInfoStr) {
+ Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
+ Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
+ Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
+
+ if (room) {
+ result.room = room
+ }
+ if (hall) {
+ result.hall = hall
+ }
+ if (toilet) {
+ result.toilet = toilet
+ }
+ }
+
+ def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
+ if (areaStr) {
+ result.area = areaStr.toDouble()
+ }
+ def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
+ result.direction = DataCrawlUtils.parseDirection(directionStr)
+
+ def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
+ if (buildDateStr && buildDateStr.contains("/")) {
+ def arrs = buildDateStr.split("\\/")
+ if (arrs && arrs.size() == 2) {
+ result.build_date = arrs[0]
+ result.estate_type = arrs[1]
+ }
+ }
+
+ result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
+ if (result.floor_info && result.floor_info.contains("共")) {
+ def height = result.floor_info
+ if (result.floor_info.contains("(")) {
+ height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
+ }
+ height = height.replace("共", "").replace("层", "")
+ if (height) {
+ result.height = height.toInteger()
+ }
+ }
+
+
+ if (result.area && result.total_price) {
+ result.avg_price = (result.total_price / result.area).toInteger()
+ }
+
+ def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
+ result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
+
+
+ def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
+
+
+ }
+ }
+
+ def getDocument(url, driver){
+ def stop = false
+ def error = 0
+ def doc = null
+ while(!stop){
+ try{
+ driver.get(url)
+ doc = Jsoup.parse(driver.getPageSource());
+ Thread.sleep(5000)
+ def els = JQueryUtils.select(".list-main .list-left .list", doc)
+ if(!els){
+ els = JQueryUtils.select(".props-main", doc)
+ }
+ if(els && els.size()>0){
+ stop = true;
+ }else{
+ error++
+ driver.manage().deleteAllCookies()
+ println "[${error}] error: "+url
+ Thread.sleep(5000)
+ }
+ }catch(Exception e){
+ error++
+ }
+ if(error>=20){
+ stop = true;
+ }
+ }
+ return doc
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy
new file mode 100644
index 000000000..85060a709
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/AnjukeToDB.groovy
@@ -0,0 +1,631 @@
+package com.fangjia.script.gj.crawl
+
+import com.fangjia.core.util.BaiDuApiUtils
+import com.fangjia.core.util.GeoUtils
+import com.fangjia.script.commData.CommonDatautils
+import com.fangjia.script.linxsh.util.AddressUtil
+import com.fangjia.script.util.CommonUtils
+import com.fangjia.script.util.DateUtils
+import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.MongoUtils
+import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.rabbitmq.client.impl.AMQImpl
+import net.sf.json.JSONObject
+import org.bson.types.ObjectId
+
+class AnjukeToDB {
+
+ DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
+
+ static main(args) {
+ def task = new AnjukeToDB();
+ task.run();
+// task.addNoMatch()
+// task.updateEstateType()
+// task.gaodeRangeData()
+ }
+
+ def updateEstateType(){
+ def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1)
+ for(def row : datas){
+ def anjukeId = row['A']
+ def obj = findAnjukeDistrictDetail(anjukeId)
+ if(obj) {
+ def type = getEstatType(obj.extend.type)
+ def commId = row['L']
+ if(!commId){
+ commId = row['G']
+ }
+ if(type){
+ if(type=="别墅"){
+ def udata = new BasicDBObject()
+ udata.put("estate_type2", type)
+ udata.put("m_date", new Date())
+ println commId
+ DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata))
+ }
+ }else{
+// def udata = new BasicDBObject()
+// udata.put("status", 1)
+// udata.put("name_cat",11)
+// udata.put("m_date", new Date())
+// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata))
+
+ }
+ }
+ }
+ }
+
+ def gaodeRangeData(){
+ def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1)
+ def index = 0
+ for(def d : datas){
+ index++
+ println index
+ def personName = d['AB']
+ if(personName){
+ def commid = d['J']
+ if(commid){
+ def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district")
+ if(obj){
+ updateDist(d, obj._id.toString())
+ }
+ }else{
+ saveGaode(d)
+ }
+ }else{
+ def commid = d['J']
+ updateDist(d, commid)
+ }
+
+ }
+ }
+
+ def saveGaode(d){
+ def oldName = d['E']
+ def distName = nameFileter(d['C'], oldName)
+ def udata = new BasicDBObject()
+ def alias = []
+
+ def aliasStr = d['F']
+ if(aliasStr){
+ def arrs = aliasStr.split("/")
+ for(def s : arrs){
+ alias << s
+ }
+ }
+ if(alias.size()>0){
+ udata.put("alias", alias)
+ }
+
+ def address = AddressUtil.formatAddress("重庆", d['G'])
+ if(address){
+ udata.put("address", address)
+ }
+ if(d['N']){
+ udata.put("virescence_percent", d['N'])
+ }
+
+ if(d['O']){
+ udata.put("estate_charge", d['O'])
+ }
+ if(d['Q']){
+ udata.put("cubage_percent", d['Q'])
+ }
+ if(d['R']){
+ udata.put("developer", d['R'])
+ }
+
+ if( d['S']){
+ udata.put("build_area", d['S'])
+ }
+
+ if(d['T']){
+ def t = d['T'].substring(0, 3)
+ udata.put("build_date", t)
+ }
+
+ if(d['V'] && d['V'].isNumber()){
+ udata.put("car_site_count", d['V'].toInteger())
+ }
+
+ if(d['Y']){
+ def build_type = getBuildType2(d['Y'])
+ if(build_type){
+ udata.put("build_type", build_type)
+ }
+ }
+
+ if(d['Z']){
+ udata.put("property_life", d['Z'])
+ }
+
+ def lat = null
+ def lng = null
+ if(d['H']){
+ lat = d['H']
+ lng = d['I']
+ // 高德转百度
+ def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
+ def lng2 = baidu.x
+ def lat2 = baidu.y
+
+ Long dLat2 = lat2 * 10000000000L
+ Long dLng2 = lng2 * 10000000000L
+ udata.put("lat2", dLat2)
+ udata.put("lng2", dLng2)
+ Long dlat = lat.toDouble() * 10000000000L
+ Long dlng = lng.toDouble() * 10000000000L
+ udata.put("lat", dlat)
+ udata.put("lng", dlng)
+ }
+ def type = "普通住宅"
+ if(distName.contains("别墅")) {
+ type = "别墅"
+ }else if(distName.contains("商住")){
+ type = "商住"
+ }
+
+ if(udata){
+ udata.put("name_cat",0)
+ udata.put("status",0)
+ udata.put("cat","district")
+ udata.put("c_date", new Date())
+ udata.put("m_date", new Date())
+ udata.put("city", "重庆")
+ udata.put("region",d['C'])
+ udata.put("name", distName)
+ udata.put("estate_type2",type)
+
+ def markMap = [:]
+ markMap.put("data_source", "机器入库")
+ udata.put("mark_map",markMap)
+ DbUtils.seaweed.save(udata);
+
+ def stop = false
+ while (!stop){
+ if(udata._id.toString()){
+ addPrice(udata)
+ stop = true;
+ }
+ }
+ }
+ }
+
+ def updateDist(d, commid){
+
+ if(commid){
+ def dist = CommonDatautils.queryDistrictById(commid)
+ if(dist){
+ def oldName = d['E']
+ def distName = nameFileter(d['C'], oldName)
+ def udata = new BasicDBObject()
+ if(!distName.contains("号")){
+ def alias = dist.alias
+ if(!alias){
+ alias = []
+ }
+ if(!alias.contains(distName) && dist.name!=distName){
+ alias << distName
+ }
+ def aliasStr = d['F']
+ if(aliasStr){
+ def arrs = aliasStr.split("/")
+ for(def s : arrs){
+ alias << s
+ }
+ }
+ if(alias.size()>0 && alias!=dist.alias){
+ udata.put("alias", alias)
+ }
+ }
+
+ if(dist.address && !dist.address.contains("号")){
+ def address = AddressUtil.formatAddress("重庆", d['G'])
+ if(address){
+ udata.put("address", address)
+ }
+ }
+
+ if(!dist.virescence_percent && d['N']){
+ udata.put("virescence_percent", d['N'])
+ }
+
+ if(!dist.estate_charge && d['O']){
+ udata.put("estate_charge", d['O'])
+ }
+ if(!dist.cubage_percent && d['Q']){
+ udata.put("cubage_percent", d['Q'])
+ }
+ if(!dist.developer && d['R']){
+ udata.put("developer", d['R'])
+ }
+
+ if(!dist.build_area && d['S']){
+ udata.put("build_area", d['S'])
+ }
+
+ if(!dist.build_date && d['T']){
+ if(d['T'].size()>=4){
+ def t = d['T'].substring(0, 4)
+ udata.put("build_date", t)
+ }
+
+ }
+
+ if(!dist.car_site_count && d['V'] && d['V'].isNumber()){
+ udata.put("car_site_count", d['V'].toInteger())
+ }
+
+ if(!dist.build_type && d['Y']){
+ def build_type = getBuildType2(d['Y'])
+ if(build_type){
+ udata.put("build_type", build_type)
+ }
+ }
+
+ if(!dist.property_life && d['Z']){
+ udata.put("property_life", d['Z'])
+ }
+
+ def lat = null
+ def lng = null
+ if(d['H']){
+ lat = d['H']
+ lng = d['I']
+
+ def slat = dist.lat/10000000000.0
+ def slng = dist.lng/10000000000.0
+
+ def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat);
+ if(distance>1000){
+ // 高德转百度
+ def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
+ def lng2 = baidu.x
+ def lat2 = baidu.y
+
+ Long dLat2 = lat2 * 10000000000L
+ Long dLng2 = lng2 * 10000000000L
+ udata.put("lat2", dLat2)
+ udata.put("lng2", dLng2)
+ Long dlat = lat.toDouble() * 10000000000L
+ Long dlng = lng.toDouble() * 10000000000L
+ udata.put("lat", dlat)
+ udata.put("lng", dlng)
+ }
+
+ }
+
+ if(udata){
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
+ println "[修改] ${dist.name}"+ udata
+ }
+ }
+ }
+ }
+
+ def addNoMatch(){
+ def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1)
+ for(def d : datas){
+ add(d)
+ }
+ }
+ def run(){
+ def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\非人工处理.xlsx",0,1)
+ for(def data : datas){
+// if(data['K']=="镕杰"){
+ def flag = data['M']
+ if(flag && flag=="新增"){
+// add(data)
+ }else{
+ update(data)
+ }
+// }
+ }
+ }
+
+
+ def add(def row){
+ def anjukeId = row['A']
+ def obj = findAnjukeDistrictDetail(anjukeId)
+ if(obj){
+ def type = getEstatType(obj.extend.type)
+ if(type){
+ def city = obj.base.cityName
+ def region = obj.base.areaName
+ def lat2 = null
+ def lng2 = null
+ def lat = null
+ def lng = null
+ if(obj.base.lat){
+ lat2 = obj.base.lat
+ lng2 = obj.base.lng
+
+ JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu")
+ if(convertObj.getInt("status") == 1) {
+ String lo = convertObj.get("locations")
+ lng = Double.parseDouble(lo.split(",")[0])
+ lat = Double.parseDouble(lo.split(",")[1])
+ }
+ }
+
+ def distName = nameFileter(obj.base.areaName, obj.base.name)
+
+ def buildTypeStr = getBuildType(obj.base.buildTypeStr)
+
+ def address = AddressUtil.formatAddress(obj.base.city, obj.base.address)
+
+ def dist = CommonDatautils.findDistrict(city, region, distName, "district")
+ if(!dist){
+ def data = new BasicDBObject();
+ data.put("name_cat",0)
+ data.put("status",0)
+ data.put("cat","district")
+ data.put("c_date", new Date())
+ data.put("m_date", new Date())
+ data.put("city", city)
+ data.put("region",region)
+ data.put("name", distName)
+ data.put("estate_type2",type)
+ if(address){
+ data.put("address", obj.base.address)
+ }
+
+ if(lat){
+ Long alat = lat*10000000000L
+ Long alng = lng*10000000000L
+ data.put("lat", alat)
+ data.put("lng", alng)
+ Long alat2 = lat2.toDouble()*10000000000L
+ Long alng2 = lng2.toDouble()*10000000000L
+ data.put("lat2",alat2)
+ data.put("lng2",alng2)
+ }
+ data.put("finish_building_date",obj.extend.completionTime)
+ if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){
+ data.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
+ }
+ data.put("build_area", obj.extend.totalArea)
+
+ data.put("virescence_percent", obj.extend.landscapingRatio)
+ data.put("build_type", buildTypeStr)
+ data.put("business_quarter",obj.base.tradingAreaName)
+
+
+ def markMap = [:]
+ markMap.put("data_source", "机器入库")
+ data.put("mark_map",markMap)
+ DbUtils.seaweed.save(data);
+
+ def stop = false
+ while (!stop){
+ if(data._id.toString()){
+ addPrice(data)
+ stop = true;
+ }
+ }
+
+ }
+ }else{
+ println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type
+ }
+ }else{
+ println anjukeId+" 没找到"
+ }
+
+ }
+ def update(def row){
+ def commId = row['L']
+ if(!commId){
+ commId = row['G']
+ }
+
+ def dist = CommonDatautils.queryDistrictById(commId)
+ def anjukeId = row['A']
+ def obj = findAnjukeDistrictDetail(anjukeId)
+ if(obj && dist){
+ def udata = new BasicDBObject()
+
+ def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address)
+ if(address){
+ if(!dist.address){
+ if(obj.base.address){
+ udata.put("address",obj.base.address)
+ }
+ }else{
+ if(dist.address.contains("号")){
+ if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){
+ def addr_alias = dist.addr_alias
+ if(!addr_alias){
+ addr_alias = []
+ }
+ if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){
+ addr_alias << obj.base.address
+ udata.put("addr_alias", addr_alias)
+ }
+ }
+ }else{
+ udata.put("address",obj.base.address)
+ }
+ }
+ }
+
+ def distName = nameFileter(obj.base.areaName, obj.base.name)
+ if(!distName.contains("号")){
+ def alias = dist.alias
+ if(!alias){
+ alias = []
+ }
+ if(alias.contains(distName)){
+ alias << distName
+ }
+ if(alias.size()>0 && alias!=dist.alias){
+ udata.put("alias", alias)
+ }
+ }
+
+ if(!dist.build_type){
+ def buildTypeStr = getBuildType(obj.base.buildTypeStr)
+ if(buildTypeStr){
+ udata.put("build_type",buildTypeStr)
+ }
+
+ }
+ if(!dist.household_count){
+ if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){
+ udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
+ }
+ }
+ if(!dist.finish_building_date){
+ if(obj.extend.completionTime){
+ udata.put("finish_building_date",obj.extend.completionTime)
+ }
+ }
+
+ if(!dist.virescence_percent){
+ if(obj.extend.landscapingRatio){
+ udata.put("virescence_percent", obj.extend.landscapingRatio)
+ }
+ }
+ if(!dist.business_quarter){
+ if(obj.base.tradingAreaName){
+ udata.put("business_quarter",obj.base.tradingAreaName)
+ }
+ }
+ def lat2 = null
+ def lng2 = null
+ if(dist.lat2){
+ lat2 = dist.lat2/10000000000.0
+ lng2 = dist.lng2/10000000000.0
+ }
+ if(dist.lat && !lat2){
+ // 高德转百度
+ def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3)
+ lng2 = baidu.x
+ lat2 = baidu.y
+
+ Long dLat2 = lat2 * 10000000000L
+ Long dLng2 = lng2 * 10000000000L
+ udata.put("lat2", dLat2)
+ udata.put("lng2", dLng2)
+ }
+
+ if(obj.base.lat) {
+ def alat2 = obj.base.lat.toDouble()
+ def alng2 = obj.base.lng.toDouble()
+ def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2);
+ if(distance>1000){
+ println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................"
+
+ def lng = null
+ def lat = null
+ JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu")
+ if(convertObj.getInt("status") == 1) {
+ String lo = convertObj.get("locations")
+ lng = Double.parseDouble(lo.split(",")[0])
+ lat = Double.parseDouble(lo.split(",")[1])
+ }
+
+ Long blat2 = alat2 *10000000000L
+ Long blng2 = alng2 *10000000000L
+ Long gLat2 = lat *10000000000L
+ Long gLng2 = lng *10000000000L
+
+ udata.put("lat2", blat2)
+ udata.put("lng2", blng2)
+ udata.put("lat", gLat2)
+ udata.put("lng", gLng2)
+ }
+ }
+
+ if(udata){
+ udata.put("m_date", new Date())
+ DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
+ println "[修改] ${dist.name}"+ udata
+ }
+
+ }
+ }
+
+ def findAnjukeDistrictDetail(anjukeId){
+ def q = new BasicDBObject()
+ try{
+ q.put("_id",new ObjectId(anjukeId))
+ return detail.findOne(q)
+ }catch(Exception e){
+ print "[查询出错]anjukeId:"+anjukeId
+ }
+ return null
+ }
+
+ def nameFileter(def region, def name){
+ name = name.replace("·","").replace(".","").replace("|","")
+ name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
+ name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
+ name = name.replace("(${region})","").replace("(","").replace(")","")
+ return name;
+ }
+
+ def getEstatType(type){
+ if(type){
+ if(type=="公寓住宅"){
+ type = "普通住宅"
+ }else if(type.contains("商住")){
+ type = "商住"
+ }else if(type.contains("别墅")){
+ type = "别墅"
+ }else{
+ type = null
+ }
+ }
+ return type
+ }
+
+ def getBuildType(def buildType){
+ def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
+ def list = []
+ if(buildType){
+ def arrs = buildType.split("\\|")
+ for(def s : arrs){
+ if(types.contains(s)){
+ list << s
+ }
+ }
+ if(list){
+ return list.join(",")
+ }
+ }
+ return null
+ }
+
+ def getBuildType2(def buildType){
+ def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
+ def list = []
+ if(buildType){
+ def arrs = buildType.split(",| ")
+ for(def s : arrs){
+ if(types.contains(s)){
+ list << s
+ }
+ }
+ if(list){
+ return list.join(",")
+ }
+ }
+ return null
+ }
+
+ def addPrice(dist){
+ def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01")
+ def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25")
+ def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
+
+ for(def weekend : weekends){
+ CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
+ }
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy
new file mode 100644
index 000000000..465ae5f04
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DataCrawlUtils.groovy
@@ -0,0 +1,250 @@
+package com.fangjia.script.gj.crawl
+
+import com.fangjia.script.util.DateUtils
+import org.apache.commons.lang.StringUtils
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+class DataCrawlUtils {
+ static final Map DEAL_DATE_RESOLVERS = [:]
+ static final Map ROOM_RESOLVERS = [:]
+ static final Map HALL_RESOLVERS = [:]
+ static final Map TOILET_RESOLVERS = [:]
+ static final Map TOTAL_PRICE_RESOLVERS = [:]
+ static final Map UNIT_PRICE_RESOLVERS = [:]
+ static final Map AREA_RESOLVERS = [:]
+ static final Map HEIGHT_RESOLVERS = [:]
+ static final Map FLOOR_RESOLVERS = [:]
+ static final List DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
+ private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>()
+
+ static {
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
+ return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
+ })
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
+ str = "20" + str
+ return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
+ })
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
+ str = "20" + str
+ return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
+ })
+
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
+ return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
+ })
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
+ return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
+ })
+
+ ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
+ str = str.replaceAll("室|房", '').trim()
+ return str.toInteger()
+ })
+
+ HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
+ str = str.replace("厅", '').trim()
+ return str.toInteger()
+ })
+ TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
+ str = str.replace("卫", '').trim()
+ return str.toInteger()
+ })
+ TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
+ str = str.replace("万", '').trim()
+ return (str.toDouble() * 10000).toInteger()
+ })
+
+ UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
+ str = str.replace("元/㎡", '').trim()
+ return str.toDouble().toInteger()
+ })
+ UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
+ str = str.replace("元/m?", '').trim()
+ return str.toDouble().toInteger()
+ })
+ UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
+ str = str.replace("元/平", '').trim()
+ return str.toDouble().toInteger()
+ })
+ UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
+ str = str.replace("元", '').trim()
+ return str.toDouble().toInteger()
+ })
+
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
+ str = str.replace("平米", '').trim()
+ return str.toDouble()
+ })
+
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
+ str = str.replace("平方米", '').trim()
+ return str.toDouble()
+ })
+
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
+ str = str.replace("平", '').trim()
+ return str.toDouble()
+ })
+
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
+ str = str.replace("㎡", '').trim()
+ return str.toDouble()
+ })
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
+ str = str.replace("m?", '').trim()
+ return str.toDouble()
+ })
+ HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
+ str = str.replace("共", '').replace('层', '')
+ return str.toInteger()
+ })
+
+ HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
+ str = str.replace("总", '').replace('层', '')
+ return str.toInteger()
+ })
+
+ HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
+ def split = str.split("/")
+ def n1 = split[0].toInteger()
+ def n2 = split[1].toInteger()
+ if (n1 <= n2) {
+ return n2
+ }
+ return null
+ })
+
+
+ FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
+ def split = str.split("/")
+ def n1 = split[0].toInteger()
+ def n2 = split[1].toInteger()
+ if (n1 <= n2) {
+ return n1
+ }
+ return n1
+ })
+
+ FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
+ str = str.replace("第", "").replaceAll("层|楼", "")
+ return str.toInteger()
+ })
+
+ FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
+ FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
+ FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
+ FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
+
+ }
+
+ /**
+ * 解析室数
+ * @param content
+ * @return
+ */
+ static Integer parseRoom(String content) {
+ return parse(content, ROOM_RESOLVERS)
+ }
+ /**
+ * 解析厅数
+ * @param content
+ * @return
+ */
+ static Integer parseHall(String content) {
+ return parse(content, HALL_RESOLVERS)
+ }
+ /**
+ * 解析厅数
+ * @param content
+ * @return
+ */
+ static Integer parseToilet(String content) {
+ return parse(content, TOILET_RESOLVERS)
+ }
+
+ static Object parse(String content, Map resolvers) {
+ if (content != null) {
+ content = content.replaceAll("\\s", "")
+ }
+ for (def resolver : resolvers.entrySet()) {
+ String match = match(content, resolver.getKey())
+ if (StringUtils.isNotBlank(match)) {
+ match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
+ return resolver.getValue()(match)
+ }
+ }
+ return null
+ }
+
+ static String match(String str, Pattern pattern) {
+ if (str == null) {
+ return null
+ }
+ Matcher matcher = pattern.matcher(str)
+ if (matcher.find()) {
+ return matcher.group()
+ }
+ return null
+ }
+
+ static String parseDirection(String directionDesc) {
+ if (directionDesc == null) {
+ return null
+ }
+ directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
+ StringBuffer sb = new StringBuffer()
+ for (int i = 0; i < directionDesc.length(); i++) {
+ String c = directionDesc.substring(i, i + 1)
+ if (DIRECTIONS.contains(c)) {
+ if (directionDesc.substring(i).startsWith(c + "#_#")) {
+ sb.append(c + " ")
+ } else {
+ sb.append(c)
+ }
+ }
+ }
+ for (String d : DIRECTIONS) {
+ if (sb.toString().startsWith(d)) {
+ return d
+ }
+ }
+ for (String d : DIRECTIONS) {
+ if (sb.toString().contains(d)) {
+ return d
+ }
+ }
+ return null
+
+ }
+
+ static String parseFitment(String fitmentDesc) {
+ if (fitmentDesc == null) {
+ return null
+ }
+ fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
+ if (fitmentDesc.equals("非毛坯")) {
+ return null
+ }
+ Iterator>> it = FITMENTS_TYPES_MAPS
+ .entrySet().iterator()
+ Map.Entry> entry
+ while (it.hasNext()) {
+ entry = it.next()
+ if (fitmentDesc.contains(entry.getKey())) {
+ return entry.getKey()
+ }
+ for (String fitmentKey : entry.getValue()) {
+ if (fitmentDesc.contains(fitmentKey)) {
+ return entry.getKey()
+ }
+ }
+ }
+ return null
+
+ }
+
+
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy
new file mode 100644
index 000000000..02fc9458d
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/DealPriceUtils.groovy
@@ -0,0 +1,347 @@
+package com.fangjia.script.gj.crawl
+
+import com.fangjia.script.util.DateUtils
+import org.apache.commons.lang.StringUtils
+
+import java.util.regex.Matcher
+import java.util.regex.Pattern
+
+/**
+ * 鎴愪氦鏁版嵁瑙f瀽宸ュ叿绫�
+ */
+class DealPriceUtils {
+
+ static final Map DEAL_DATE_RESOLVERS = [:]
+ static final Map ROOM_RESOLVERS = [:]
+ static final Map HALL_RESOLVERS = [:]
+ static final Map TOILET_RESOLVERS = [:]
+ static final Map TOTAL_PRICE_RESOLVERS = [:]
+ static final Map UNIT_PRICE_RESOLVERS = [:]
+ static final Map AREA_RESOLVERS = [:]
+ static final Map HEIGHT_RESOLVERS = [:]
+ static final Map FLOOR_RESOLVERS = [:]
+ static final List DIRECTIONS = ["涓滃寳", "瑗垮寳", "涓滃崡", "瑗垮崡", "鍗楀寳", "涓滆タ", "涓�", "鍗�", "瑗�", "鍖�"]
+ private static final HashMap> FITMENTS_TYPES_MAPS = new HashMap>()
+
+ static {
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
+ return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
+ })
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
+ str = "20" + str
+ return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
+ })
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
+ str = "20" + str
+ return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
+ })
+
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
+ return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
+ })
+ DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
+ return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
+ })
+
+ ROOM_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓(瀹鎴�)"), { String str ->
+ str = str.replaceAll("瀹鎴�", '').trim()
+ return str.toInteger()
+ })
+
+ HALL_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍘�"), { String str ->
+ str = str.replace("鍘�", '').trim()
+ return str.toInteger()
+ })
+ TOILET_RESOLVERS.put(Pattern.compile("[\\d+|涓�|浜寍涓墊鍥泑浜攟涓鍗�"), { String str ->
+ str = str.replace("鍗�", '').trim()
+ return str.toInteger()
+ })
+ TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?涓�"), { String str ->
+ str = str.replace("涓�", '').trim()
+ return (str.toDouble() * 10000).toInteger()
+ })
+
+ UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/銕�"), { String str ->
+ str = str.replace("鍏�/銕�", '').trim()
+ return str.toDouble().toInteger()
+ })
+ UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/m虏"), { String str ->
+ str = str.replace("鍏�/m虏", '').trim()
+ return str.toDouble().toInteger()
+ })
+ UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�/骞�"), { String str ->
+ str = str.replace("鍏�/骞�", '').trim()
+ return str.toDouble().toInteger()
+ })
+ UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?鍏�"), { String str ->
+ str = str.replace("鍏�", '').trim()
+ return str.toDouble().toInteger()
+ })
+
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞崇背"), { String str ->
+ str = str.replace("骞崇背", '').trim()
+ return str.toDouble()
+ })
+
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞虫柟绫�"), { String str ->
+ str = str.replace("骞虫柟绫�", '').trim()
+ return str.toDouble()
+ })
+
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?骞�"), { String str ->
+ str = str.replace("骞�", '').trim()
+ return str.toDouble()
+ })
+
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?銕�"), { String str ->
+ str = str.replace("銕�", '').trim()
+ return str.toDouble()
+ })
+ AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m虏"), { String str ->
+ str = str.replace("m虏", '').trim()
+ return str.toDouble()
+ })
+ HEIGHT_RESOLVERS.put(Pattern.compile("鍏盶\d+灞�"), { String str ->
+ str = str.replace("鍏�", '').replace('灞�', '')
+ return str.toInteger()
+ })
+
+ HEIGHT_RESOLVERS.put(Pattern.compile("鎬籠\d+灞�"), { String str ->
+ str = str.replace("鎬�", '').replace('灞�', '')
+ return str.toInteger()
+ })
+
+ HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
+ def split = str.split("/")
+ def n1 = split[0].toInteger()
+ def n2 = split[1].toInteger()
+ if (n1 <= n2) {
+ return n2
+ }
+ return null
+ })
+
+
+ FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
+ def split = str.split("/")
+ def n1 = split[0].toInteger()
+ def n2 = split[1].toInteger()
+ if (n1 <= n2) {
+ return n1
+ }
+ return n1
+ })
+
+ FLOOR_RESOLVERS.put(Pattern.compile("绗琝\d+(灞倈妤�)"), { String str ->
+ str = str.replace("绗�", "").replaceAll("灞倈妤�", "")
+ return str.toInteger()
+ })
+
+ FITMENTS_TYPES_MAPS.put("姣涘澂", ["姣涘澂", "娓呮按", "鏃犺淇�", "鐙爧姣涘澂", "姣涜儦", "鐧藉澂", "鏃ц", "鍧�", "鍘熸埧", "姣涘潖"])
+ FITMENTS_TYPES_MAPS.put("绠�瑁�", ["绠�瑁�", "鏅�氳淇�", "鏅��", "绠�鍗曡淇�", "绠�鏄撹淇�", "鏅", "涓�鑸淇�", "涓�鑸�", "绠�", "绮楄淇�"])
+ FITMENTS_TYPES_MAPS.put("绮捐", ["绮捐", "鍏ㄨ淇�", "鍏ㄨ", "涓瓑瑁呬慨", "涓", "涓。瑁呬慨", "绮捐嚧瑁呬慨", "涓。", "涓瓑", "绮�", "涓慨", "浼樿川瑁呬慨", "鏃跺皻瑁呬慨", "鑹", "涓珮鎸�"])
+ FITMENTS_TYPES_MAPS.put("璞", ["璞崕瑁呬慨", "楂樻。", "璞崕", "璞", "楂樼骇瑁呬慨", "楂樿", "濂緢"])
+
+ }
+ /**
+ * 瑙f瀽鎴愪氦鏃ユ湡
+ * @param content
+ * @return
+ */
+ static Date parseDealDate(String content) {
+ return parse(content, DEAL_DATE_RESOLVERS)
+ }
+
+ /**
+ * 瑙f瀽瀹ゆ暟
+ * @param content
+ * @return
+ */
+ static Integer parseRoom(String content) {
+ return parse(content, ROOM_RESOLVERS)
+ }
+ /**
+ * 瑙f瀽鍘呮暟
+ * @param content
+ * @return
+ */
+ static Integer parseHall(String content) {
+ return parse(content, HALL_RESOLVERS)
+ }
+ /**
+ * 瑙f瀽鍘呮暟
+ * @param content
+ * @return
+ */
+ static Integer parseToilet(String content) {
+ return parse(content, TOILET_RESOLVERS)
+ }
+
+ /**
+ * 瑙f瀽鎬讳环
+ * @param content
+ * @return
+ */
+ static Integer parseTotalPrice(String content) {
+ return parse(content, TOTAL_PRICE_RESOLVERS)
+ }
+ /**
+ * 瑙f瀽鍗曚环
+ * @param content
+ * @return
+ */
+ static Integer parseUnitPrice(String content) {
+ return parse(content, UNIT_PRICE_RESOLVERS)
+ }
+ /**
+ * 瑙f瀽闈㈢Н
+ * @param content
+ * @return
+ */
+ static Double parseArea(String content) {
+ return parse(content, AREA_RESOLVERS)
+ }
+ /**
+ * 瑙f瀽妤煎眰
+ * @param content
+ * @return
+ */
+ static Integer parseFloor(String content) {
+ return parse(content, FLOOR_RESOLVERS)
+ }
+ /**
+ * 瑙f瀽妤奸珮
+ * @param content
+ * @return
+ */
+ static Integer parseHeight(String content) {
+ return parse(content, HEIGHT_RESOLVERS)
+ }
+
+ /**
+ * 鏍煎紡鍖栧皬鍖哄悕
+ * @param name
+ * @return
+ */
+ static String formatName(String name) {
+ name = name.replace("-鍟嗕綇", "").replace("-鍒", "")
+ //姹囩瀹跺洯骞冲畨閲岋紙杩侊級2015-2017
+ name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "")
+ //鍥借锤澶╂偊涓�鏈�2017
+ name = name.replaceAll("\\d\\d\\d\\d\$", "")
+ if (name ==~ /.+(\(|锛�)[\d涓�浜屼笁鍥涗簲鍏竷鍏節鍗佷笢鍗楄タ鍖梶\w]+(鏈焲鍖�)(\)|锛�)$/) {
+ name = name.replaceAll("\\(|锛坾\\)|锛�", '')
+ } else if (name ==~ /.+(\(|锛�)鍒(\)|锛�)$/) {
+ name = name.replaceAll("\\(|锛坾\\)|锛�", '')
+ } else if (name ==~ /.+(\(|锛�).+(\)|锛�)$/) {
+ name = name.replaceAll("(\\(|锛�).+(\\)|锛�)", '')
+ } else if (name.contains("路") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) {
+ println("---------------------------->" + name)
+ name = name.replaceAll("路|-", '')
+ println(name)
+ }
+ if (name ==~ /.{2,}璺�.{3,}$/ && !(name ==~ /.{2,}璺�.+鍙�/) && !(name ==~ /.{2,}璺�.+寮�/) && !(name ==~ /.*灏忓尯$/)) {
+ name = name.replaceAll("^.{2,}璺�", "")
+ }
+ return name
+ }
+
+
+ static Object parse(String content, Map resolvers) {
+ if (content != null) {
+ content = content.replaceAll("\\s", "")
+ }
+ for (def resolver : resolvers.entrySet()) {
+ String match = match(content, resolver.getKey())
+ if (StringUtils.isNotBlank(match)) {
+ match = match.replace("涓�","1").replace("浜�","2").replace("涓�","3").replace("鍥�","4").replace("浜�","5").replace("涓�","2")
+ return resolver.getValue()(match)
+ }
+ }
+ return null
+ }
+
+ /**
+ * 鍖归厤绗﹀悎鏉′欢鐨勫瓧绗︿覆
+ *
+ * @param str
+ * @param pattern
+ * @return
+ */
+ static String match(String str, Pattern pattern) {
+ if (str == null) {
+ return null
+ }
+ Matcher matcher = pattern.matcher(str)
+ if (matcher.find()) {
+ return matcher.group()
+ }
+ return null
+ }
+
+ static String parseFitment(String fitmentDesc) {
+ if (fitmentDesc == null) {
+ return null
+ }
+ fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
+ if (fitmentDesc.equals("闈炴瘺鍧�")) {
+ return null
+ }
+ Iterator>> it = FITMENTS_TYPES_MAPS
+ .entrySet().iterator()
+ Map.Entry> entry
+ while (it.hasNext()) {
+ entry = it.next()
+ if (fitmentDesc.contains(entry.getKey())) {
+ return entry.getKey()
+ }
+ for (String fitmentKey : entry.getValue()) {
+ if (fitmentDesc.contains(fitmentKey)) {
+ return entry.getKey()
+ }
+ }
+ }
+ return null
+
+ }
+
+ static String parseDirection(String directionDesc) {
+ if (directionDesc == null) {
+ return null
+ }
+ directionDesc = directionDesc.replaceAll("(\\(|锛坾銆妡\"|鈥渱銆恷銆�).+?(\\)|銆媩锛墊\"|鈥潀銆憒銆�)", "#_#")
+ StringBuffer sb = new StringBuffer()
+ for (int i = 0; i < directionDesc.length(); i++) {
+ String c = directionDesc.substring(i, i + 1)
+ if (DIRECTIONS.contains(c)) {
+ if (directionDesc.substring(i).startsWith(c + "#_#")) {
+ sb.append(c + " ")
+ } else {
+ sb.append(c)
+ }
+ }
+ }
+ for (String d : DIRECTIONS) {
+ if (sb.toString().startsWith(d)) {
+ return d
+ }
+ }
+ for (String d : DIRECTIONS) {
+ if (sb.toString().contains(d)) {
+ return d
+ }
+ }
+ return null
+
+ }
+
+
+ static String parsePrice(String priceStr) {
+ def pattern =Pattern.compile("\\d+(\\.\\d+)?涓�")
+ String match = match(priceStr, pattern)
+ }
+
+ static void main(String[] args) {
+ println parsePrice("椤剁娆e洯瑗胯嫅2瀹�1鍘�1鍗�492.3涓�")
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy
new file mode 100644
index 000000000..845ab47d0
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy
@@ -0,0 +1,167 @@
+package com.fangjia.script.gj.crawl
+
+import com.fangjia.script.crawl.JQueryUtils
+import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
+import com.fangjia.script.thirdPriceCrawl.Utils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.mongodb.DBObject
+import org.jsoup.Jsoup
+import org.openqa.selenium.By
+import org.openqa.selenium.WebDriver
+import org.openqa.selenium.chrome.ChromeDriver
+
+class ZhugeFangDistCrawl {
+
+ public static final SHORT_DATE = Utils.getLastMonth();
+ def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
+
+ static main(args) {
+ def task = new ZhugeFangDistCrawl();
+ task.run();
+ }
+ def run(){
+ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
+ System.setProperty("http.nonProxyHosts", "localhost");
+ def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
+ WebDriver driver = new ChromeDriver()
+ driver.manage().deleteAllCookies()
+ getRegions(driver, url)
+ }
+
+ def getRegions(driver, url) {
+ for(def i=1;i<=11;i++){
+ def newUrl = url+"page${i}/"
+ println newUrl
+ def doc = getDocument(newUrl, driver)
+ def total = parseList(doc)
+
+// def next = JQueryUtils.select("a:contains(下一页)", doc)
+// if(next){
+// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
+// }
+
+ }
+
+ }
+
+ def parseList(doc){
+ def city= "福州"
+ def els = JQueryUtils.select("#listTableBox li", doc)
+ if (els) {
+ for(def e : els){
+ def name = e.select(".house-name").text()
+ def str = e.select(".house-adr").text()
+ def address = null
+ if(str){
+ if(str.contains("]")){
+ address = str.substring(str.indexOf("]")+1).trim()
+ }else{
+ address = str
+ }
+ }
+ def regionStr = null
+ if(str && str.contains("]") && str.contains("[")){
+ regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]"))
+ }
+
+ def region = null
+ def block = null
+ if(regionStr){
+ def arrs = regionStr.split("-")
+ if(arrs && arrs.size()>0){
+ if(arrs.size()==2){
+ block = arrs[1].trim()
+ }
+ if(arrs.size()>=1){
+ region = arrs[0].trim()
+ }
+
+ }
+ }
+ def links = e.select(".house-name a")
+ def href = null
+ if(links){
+ href = links[0].attr("href")
+ }
+ def priceStr = e.select(".price-number").text()
+ def price = null
+ if(priceStr && !priceStr.contains("暂无")){
+ if(priceStr.contains("元")){
+ priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim()
+ }
+ if(priceStr && priceStr.isNumber()){
+ price = priceStr.toInteger()
+ }
+ }
+ def obj = findByParams(city, region, name)
+ if(obj){
+ if(price){
+ DBObject udata2 = new BasicDBObject()
+ udata2.put("avg_price", price)
+ udata2.put("m_date", new Date())
+ println "[upeate] ${region} "+name
+ thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
+ }
+ }else{
+ def udata = new BasicDBObject()
+ udata.put("city", city)
+ udata.put("region", region)
+ udata.put("block", block)
+ udata.put("url", href)
+ udata.put("address", address)
+ udata.put("name", name)
+ udata.put("avg_price", price)
+ udata.put("s_date", SHORT_DATE)
+ udata.put("c_date", new Date())
+ udata.put("m_date", new Date())
+ udata.put("source", "诸葛找房")
+ thirdPartyColl.save(udata)
+ println "[save] ${region} "+name+" "+priceStr
+ }
+ }
+ }
+ return els.size()
+ }
+
+
+ def findByParams(city, region, name){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("region", region)
+ q.put("name", name)
+ q.put("source", "诸葛找房")
+ q.put("s_date", SHORT_DATE)
+ return thirdPartyColl.findOne(q);
+ }
+
+
+ def getDocument(url, driver){
+ def stop = false
+ def error = 0
+ def doc = null
+ while(!stop){
+ try{
+ driver.get(url)
+ doc = Jsoup.parse(driver.getPageSource());
+ Thread.sleep(10000)
+ def els = JQueryUtils.select("#listTableBox li", doc)
+ if(els && els.size()>0){
+ stop = true;
+ }else{
+ error++
+ driver.manage().deleteAllCookies()
+ println "[${error}] error: "+url
+ Thread.sleep(5000)
+ }
+ }catch(Exception e){
+ error++
+ }
+ if(error>=20){
+ stop = true;
+ }
+ }
+ return doc
+ }
+
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy
index 71657eea3..cbee24228 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy
@@ -172,7 +172,8 @@ class NameProcess {
if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){
return null
}
- if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){
+ //|| name.endsWith("组")
+ if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){
return null
}
if(name.contains("、")||name.contains("|")||name.contains(".")){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy
index bd387ed77..3a572c7ad 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy
@@ -71,7 +71,8 @@ class DistrictIndexBatch extends BaseBatch{
def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
- def citys = CommonDatautils.findAllCity();
+ def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
+ //def citys = CommonDatautils.findAllCity();
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
def code = cityCodeDatas.get(city)
@@ -86,7 +87,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913
}
if(city_avg_total_price){
- // calcute(city, dists, code, city_avg_total_price)
+ calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price)
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy
new file mode 100644
index 000000000..47c9fe748
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy
@@ -0,0 +1,123 @@
+package com.fangjia.script.gj.location
+
+import com.fangjia.core.util.GeoUtils
+import com.fangjia.script.common.DataHandleUtils
+import com.fangjia.script.gj.OpenApiUtils
+import com.fangjia.script.util.CosineSimilarAlgorithm
+import com.fangjia.script.util.ExcelUtils
+
+class DistrictMapMatch {
+
+ static main(args) {
+ def task = new DistrictMapMatch()
+ task.mapMatch()
+ }
+
+
+ def mapMatch(){
+ def index = 0
+ ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d->
+ if(!d['H']){
+ index++
+ def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
+ def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
+// if (!map || !map.lat) {
+// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
+// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
+// }
+ if(map){
+ d['F'] = map.id
+ if(map.adname){
+ d['G'] = map.adname
+ }else if(map.region){
+ d['G'] = map.region
+ }
+ d['H'] = map.name
+ d['I'] = map.address
+ if(map.type){
+ d['J'] = map.type
+ }else if(map.estateType){
+ d['J'] = map.estateType
+ }
+ d['K'] = map.lng
+ d['L'] = map.lat
+ d['M'] = map.score
+ d['N'] = map.source
+ }
+ }
+ },0)
+ }
+
+
+ def findStandardObject2(def city, def region, def name, def address, def rets) {
+ if (rets) {
+ def bestObj = [:]
+ if (rets.size() >= 1) {
+ for (def mapObj : rets) {
+ def score = 0
+ if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
+ score = 0.2
+ }
+ if(mapObj.name && name){
+ def newName = mapObj.name;
+ def names = []
+ if(mapObj.name && mapObj.name.indexOf("(")>-1){
+ def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
+ names << name1
+ def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
+ names << name2
+ }else{
+ names << newName
+ }
+ def maxNameScore = 0
+ for(def n : names){
+ double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
+ if(score1 && score1>0){
+ def nameScore = (score1 * 4)/10
+ if(nameScore > maxNameScore){
+ maxNameScore = nameScore
+ }
+ }
+ }
+
+ if(maxNameScore){
+ try{
+ maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
+ }catch(Exception e){
+ maxNameScore = 0
+ }
+
+ }
+ score = maxNameScore + score;
+ }
+
+ if(mapObj.address && address){
+ double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
+ if(addressScore && addressScore>0){
+ try{
+ addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
+ }catch(Exception e){
+ addressScore = 0
+ }
+ score = addressScore*4/10 + score
+ }
+ }
+
+ if(!bestObj){
+ mapObj.score = score
+ bestObj = mapObj
+ }else{
+ if(score > bestObj.score){
+ mapObj.score = score
+ bestObj = mapObj
+ }
+ }
+
+ }
+ }
+ return bestObj
+ }
+ return null
+
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy
index 6b4872d19..f42431df6 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy
@@ -21,10 +21,58 @@ class MatchTest {
def task = new MatchTest()
// task.run()
// task.addNum()
- task.dongguang()
+// task.dongguang()
// task.meituanDel()
+
+ task.addLevel()
}
+
+
+ def addLevel() {
+ def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
+ def weekend = DateUtils.getFirstDayOfWeek(new Date());
+ def cityLevelMap = [:]
+ for(def city : citys){
+ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
+ def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
+ def datas = addPriceData(dists, priceMap);
+ datas = calcuteLevel(datas);
+ cityLevelMap.put(city, ["price":priceMap, "level":datas])
+ println city +" 楼盘评级同步完成"
+ }
+
+
+ def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
+ ExcelUtils.multiThreadingModify(path, { row ->
+
+ def name = row['D']
+ def address = row['E']
+ def city = RegionData.formatCity(row['A'])
+
+ if(name && !row['O']){
+ if(city){
+ def map = cityLevelMap.get(city)
+ def rMap = null
+ if(map){
+ rMap = getArroundDistrictPriceByAddress(city, address, map.level)
+ }
+
+ if(rMap){
+ if(rMap.rank){
+ row['Q'] = rMap.rank
+ }
+// if(rMap.level){
+// row['Q'] = rMap.level
+// }
+ }
+ }
+ }
+
+ })
+ }
+
+
def meituanDel(){
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
def index = 0
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
index a6b739898..60ae682b9 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy
@@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
- def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
+ def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
- def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
+ def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -193,7 +193,7 @@ public class BeiKeCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
- detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
+ detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
}
}
@@ -215,7 +215,8 @@ public class BeiKeCrawlByChrome {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "贝壳找房"
- def dbo = findTrade(detail)
+ def detailUrl = e.select(".title a").attr("href")
+ def dbo = findTrade(detailUrl)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
@@ -228,7 +229,7 @@ public class BeiKeCrawlByChrome {
detail.height = floorStr.toInteger()
}
}
- detail.url = e.select(".title a").attr("href")
+ detail.url = detailUrl
detail.create_date = new Date()
detail.m_date = new Date()
println detail
@@ -313,8 +314,8 @@ public class BeiKeCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
- detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
- detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
+ detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
+ detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger()
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -336,13 +337,14 @@ public class BeiKeCrawlByChrome {
}
- def findTrade(def detail){
+ def findTrade(def detailUrl){
def q = new BasicDBObject()
q.put("city", detail.city)
- q.put("region", detail.region)
- q.put("district_name", detail.district_name)
- q.put("area", detail.area)
- q.put("total_price", detail.total_price)
+// q.put("region", detail.region)
+// q.put("district_name", detail.district_name)
+// q.put("area", detail.area)
+// q.put("total_price", detail.total_price)
+ q.put("url", detailUrl)
q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q)
@@ -358,13 +360,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
- if(e.text().contains("重庆")){
+// if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
- }
+// }
}
}
return list
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy
index 3ea9f1d4a..e11e01e8f 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy
@@ -29,7 +29,7 @@ class JdPaimaiCrawl {
task.run();
}
def run(){
- def existsIds = findAllIds()
+ def existsIds = []//findAllIds()
println "已存在:"+existsIds.size()
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
@@ -43,78 +43,80 @@ class JdPaimaiCrawl {
def getList(driver){
def list = []
- for(def i=1;i<=32;i++){
- if(i==8){
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
- }else if(i==5){
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
- }else if(i==17){
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
- }else if(i==19){
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
- }else{
- list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
- }
- }
+ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268"
+// for(def i=1;i<=32;i++){
+// if(i==8){
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
+// }else if(i==5){
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
+// }else if(i==17){
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
+// }else if(i==19){
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
+// }else{
+// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
+// }
+
+// }
def links = []
for(def url : list){
driver.get(url)
@@ -430,10 +432,10 @@ class JdPaimaiCrawl {
def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc)
if(s1) {
- def start = s1.indexOf("法院将于")
+ def start = s1.indexOf("将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
- s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
+ s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止"))
}
def dateRangs = DataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
@@ -476,6 +478,11 @@ class JdPaimaiCrawl {
}
}
+
+ if(!result.startDate){
+ def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","")
+ result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日")
+ }
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
index 6f99480ab..63ecf3c2a 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy
@@ -38,6 +38,26 @@ public class LianjiaCrawlByChrome {
// task.run1()
task.addDealRegion();
task.standardRegion()
+// task.updadteDealPriceField()
+ }
+
+ def updadteDealPriceField(){
+ def citys = CommonDatautils.findAllCity()
+ def index = 0
+ for(def city : citys){
+ index ++
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("hall", "0")
+ def count = DEAL_PRICE_COLL.count(q)
+ println "[${index}] "+city +" "+count
+ if(count && count>0){
+ def udata = new BasicDBObject()
+ udata.put("hall", 0)
+ DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
+ println "[修改]"+city +" "+ count
+ }
+ }
}
def run1(){
@@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
- def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
+ def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
- def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd")
+ def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -258,7 +278,7 @@ public class LianjiaCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
- detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
+ detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
}
}
@@ -378,8 +398,8 @@ public class LianjiaCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
- detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
- detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
+ detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
+ detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger()
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
- def citys= ["长春","吉林"]
+ def citys= ["福州"]
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
- if (!href.contains("fang.lianjia") && citys.contains(e.text())){
+ if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy
index acced0959..6a50bd201 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy
@@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON
import com.fangjia.core.util.DataUtil
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils
+import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
+import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
+import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
@@ -19,9 +22,102 @@ class PaimaiTest {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
+ public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
+ public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
+
+
static main(args) {
def task = new PaimaiTest();
- task.run();
+// task.run();
+// task.stat()
+ task.updatePaimai()
+// task.updateFieldPaimai()
+ }
+
+ def updateFieldPaimai(){
+ def q = new BasicDBObject()
+ q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
+ def index = 0
+ def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
+ for(def obj : dbcursor) {
+ index++
+ println index
+ if(obj.address && obj.address.contains("可贷款")){
+ obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
+ paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
+ }
+ }
+ }
+
+ def updatePaimai(){
+ def q = new BasicDBObject()
+ q.put("url", Pattern.compile(".*jd.com.*"))
+ def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
+ def index =0
+ for(def d : dbcursor){
+ index++
+ println index
+ if(!d.url.contains("jd.com")){
+ continue
+ }
+
+ if(!d.proince.contains("重庆")){
+ continue
+ }
+
+ println d.url
+
+ def query = new BasicDBObject()
+ query.put("url", d.url)
+ def dbo = paimai43_COLL.findOne(query);
+
+ DBObject obj = new BasicDBObject();
+ d.each{k,v->
+ obj.put(k,v)
+ }
+
+ if(dbo){
+ obj.m_date = new Date()
+ paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
+ }else{
+ obj.c_date = new Date()
+ obj.m_date = new Date()
+ try{
+ paimai43_COLL.save(obj)
+ }catch(Exception e){
+ println obj.url
+ }
+ }
+ }
+ }
+
+ def stat(){
+ def q = new BasicDBObject()
+ q.put("city", new BasicDBObject('$in', ["重庆","重庆市"]))
+ def dbcursor = paimaiResultColl.find(q)
+ def keys = []
+ for(def db : dbcursor){
+ if(db.matchCity){
+ def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName;
+ if(!keys.contains(key)){
+ keys << key
+ }
+// try{
+// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null)
+// if(jsonObject != null && jsonObject.precise!=-1) {
+// db.matchCity = jsonObject.get("city")
+// db.matchRegion = jsonObject.get("district")
+// db.matchName = jsonObject.get("name")
+// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db))
+// println db.address+" "+db.matchName
+// }
+// }catch(Exception e){
+// println "error "+db.address
+// }
+
+ }
+ }
+ println keys.size()
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy
index 8d971ee8a..926581423 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy
@@ -11,14 +11,17 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
+import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import java.text.DecimalFormat
+import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
+import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -34,6 +37,9 @@ class CZDataBatch extends BaseBatch{
DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com")
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
//final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls");
+
+ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
+
static main(args) {
runBatch(new CZDataBatch(), args);
}
@@ -41,11 +47,17 @@ class CZDataBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity();
+
citys.remove("常州");
citys.add(0, "常州")
for(def city : citys){
+
+ def weekend = DateUtils.getFirstDayOfWeek(new Date())
+ def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
+ def commids = findAllJnBankDatas(city)
+
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
- districtRun(districts);
+ districtRun(districts, statMap, commids);
def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
shops = mergeShop(shops);
@@ -112,29 +124,48 @@ class CZDataBatch extends BaseBatch{
return rMap;
}
- def districtRun(def districts){
-
+ def districtRun(def districts, def statMap, commids){
def total = districts.size();
- def index = 0;
- ExecutorService executorService = Executors.newFixedThreadPool(20)
-
+ def totalCount = new AtomicLong()
+ final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i0){
+ if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){
+ def newValue = house_floor +"-"+room_num
+ println "[${index}] "+room_num +" "+house_floor+" "+newValue
+ def udata = new BasicDBObject()
+ udata.put("room_num",newValue)
+ HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false);
+ }
+ }
+ }
+ }catch(Exception e){
+ println d._id.toString()
+ }
+
+ }
}
def export(){
def q = new BasicDBObject()
@@ -157,29 +201,378 @@ class CqfdcMatch {
return rets;
}
+ def addHouseLoudong(){
+ def q = new BasicDBObject()
+ def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
+ def keys = []
+ def index = 0
+ for(def d : dbcursor) {
+ index++
+ println index
+ if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){
+ continue
+ }
+ if(d.area.toDouble()<=25){
+ continue
+ }
+ def newRegion = RegionData.formatRegion(d.city, d.region)
+ if (newRegion) {
+ def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
+ def house_num = null
+ if (houseNumStr) {
+ house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "")
+ }
+ if (house_num) {
+ def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
+ if (unitNum) {
+ unitNum = unitNum.replace("单元", "")
+ }
+ def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num
+ if(!keys.contains(key) && d.room_num) {
+ keys << key
+ def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num)
+ if(!count){
+ def dist = null
+ if(d.match_city){
+ dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
+ }
+ def udata = new BasicDBObject()
+ udata.put("city", d.city)
+ udata.put("region", newRegion)
+ udata.put("name", d.district_name)
+ udata.put("house_num", house_num)
+ if(unitNum){
+ udata.put("house_num_unit", unitNum)
+ }
+ udata.put("room_num", d.room_num)
+ udata.put("house_area",d.area.toDouble())
+ udata.put("status", 0)
+ udata.put("from", "房谱网")
+ if(d.floor_heigth && d.floor_heigth.isNumber()){
+ udata.put("house_height", d.floor_heigth.toInteger())
+ }
+ if(d.floor && d.floor.isNumber()){
+ udata.put("house_floor", d.floor.toInteger())
+ }
+ if(dist){
+ udata.put("district_id", dist._id.toString())
+ udata.put("name_in_seaweed", dist.name)
+ }
+ if(d.address){
+ def address = AddressUtil.formatAddress(d.city, d.address)
+ if(address){
+ udata.put("address", address)
+ }
+ }
+ udata.put("c_date", new Date())
+ udata.put("m_date", new Date())
+ HOUSE_LOUDONG_LINE_COLL.save(udata)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){
+ def q = new BasicDBObject()
+ q.put("city", city)
+ q.put("region", region)
+ q.put("name", district_name)
+ q.put("house_num", house_num)
+ if(unitNum){
+ q.put("unit_num", unitNum)
+ }
+ q.put("room_num", room_num)
+ def count = HOUSE_LOUDONG_LINE_COLL.count(q)
+ return count;
+ }
+
def findAllLoundongByHouseNum(){
def q = new BasicDBObject()
- def dbcursor = loudongConn.find(q).batchSize(5000)
- def rets = []
- def map = [:]
+ def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000)
+ def keys = []
for(def d : dbcursor){
- def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num
- if(d.unit_num){
- key = key +"##"+d.unit_num
- }
- def detailMap= map.get(key)
- if(!detailMap){
- detailMap = [:]
- map.put(key,detailMap)
- }
+ def newRegion = RegionData.formatRegion(d.city, d.region)
+ if(newRegion){
+ def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
+ def house_num = null
+ if(houseNumStr){
+ house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
+ }
+ if(house_num){
+ def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
+ if(unitNum){
+ unitNum = unitNum.replace("单元","")
+ }
+ def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num
+ if(unitNum){
+ key = key +"##"+unitNum
+ }
+ if(!keys.contains(key)){
+ keys << key
+ def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num)
+ println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}"
+ if(obj){
+ def udata = new BasicDBObject()
+ def areas = obj.areas
+ if(!areas.contains(d.area)){
+ areas << d.area
- if(detailMap.height){
+ def forms = obj.from
+ if(!forms.contains("房谱网")){
+ forms << "房谱网"
+ }
+ obj.m_date = new Date()
+ def query = new BasicDBObject("_id", obj._id)
+ LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
+
+ }
+ }else{
+ def udata = new BasicDBObject()
+ udata.put("city", d.city)
+ udata.put("region", newRegion)
+ udata.put("name", d.district_name)
+ udata.put("house_num", house_num)
+ if(unitNum){
+ udata.put("unit_num", unitNum)
+ }
+ if(d.address){
+ def address = AddressUtil.formatAddress(d.city, d.address)
+ if(address){
+ udata.put("address", address)
+ }
+ }
+ udata.put("areas",[d.area])
+ udata.put("from",["房谱网"])
+ if(d.floor_heigth && d.floor_heigth.toString().toInteger()){
+ udata.put("floors", d.floor_heigth.toString().toInteger())
+ }
+
+ udata.put("c_date", new Date())
+ udata.put("status", 1)
+ LOUDONG_LINE_COLL.save(udata)
+ }
+ }
+ }
}
}
- return rets;
}
+ def addFdcHouseLoudong(){
+ def q = new BasicDBObject()
+ def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000)
+ def keys = []
+ def index = 0
+ for(def d : dbcursor){
+ index++
+ println index
+ if(!d.use_type){
+ continue
+ }
+ if(!d.use_type.contains("住宅")){
+ continue
+ }
+ def newRegion = d.match_region
+ if(newRegion){
+ def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
+ def house_num = null
+ if(houseNumStr){
+ house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
+ }
+ if(house_num){
+ def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num
+ if(!keys.contains(key)) {
+ keys << key
+ def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num)
+ if(!count){
+ def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
+ def dist = null
+ if(d.match_city){
+ dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
+ }
+ def floor = null
+ if(d.room_num && d.room_num.contains("-")) {
+ def arrs = d.room_num.split("-")
+ floor = arrs[0]
+ }
+ Integer room = null
+ Integer hall = null
+ if(d.room_type && d.room_type.contains("室")){
+ def roomStr = d.room_type.substring(0,d.room_type.indexOf("室"))
+ if(roomStr){
+ roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
+ room = roomStr.toInteger()
+ }
+
+ if(d.room_type.contains("厅")){
+ def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅"))
+ if(hallStr){
+ hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
+ hall = hallStr.toInteger()
+ }
+ }
+ }
+
+
+ def udata = new BasicDBObject()
+ udata.put("city", d.city)
+ udata.put("region", newRegion)
+ udata.put("name", d.name)
+ udata.put("house_num", house_num)
+ if(d.unit_num){
+ udata.put("house_num_unit", d.unit_num)
+ }
+ udata.put("room_num", d.room_num)
+ udata.put("house_area",d.area.toDouble())
+ udata.put("status", 0)
+ udata.put("from", "重庆网上房地产")
+ udata.put("house_height", maxHeight)
+ if(room){
+ udata.put("room", room)
+ }
+ if(hall){
+ udata.put("hall", hall)
+ }
+ if(floor && floor.isNumber()){
+ udata.put("house_floor", floor.toInteger())
+ }
+ if(dist){
+ udata.put("district_id", dist._id.toString())
+ udata.put("name_in_seaweed", dist.name)
+ }
+ if(d.address){
+ def address = AddressUtil.formatAddress(d.city, d.address)
+ if(address){
+ udata.put("address", address)
+ }
+ }
+ udata.put("c_date", new Date())
+ udata.put("m_date", new Date())
+ HOUSE_LOUDONG_LINE_COLL.save(udata)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public String getMatchString(String text,String pattern){
+ Pattern p=Pattern.compile(pattern);
+ Matcher m=p.matcher(text);
+ if(m.find()){
+ return m.group(0)
+ }
+ return null;
+ }
+
+ def addFdcBaseLoudong(){
+ def q = new BasicDBObject()
+ def dbcursor = loudongFdcConn.find(q).batchSize(5000)
+ def keys = []
+ for(def d : dbcursor){
+ if(!d.match_region){
+ continue
+ }
+ def newRegion = d.match_region
+
+ def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
+ def house_num = null
+ if(houseNumStr){
+ house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
+ }
+ if(house_num){
+ def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num
+ if(d.unit_num) {
+ key = key + "##" + d.unit_num
+
+ if (!keys.contains(key)) {
+ keys << key
+ def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num)
+ println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}"
+
+ def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
+ if (obj) {
+
+ def areas = obj.areas
+ if (!areas.contains(d.area)) {
+ areas << d.area
+
+ def forms = obj.from
+ if (!forms.contains("重庆网上房地产")) {
+ forms << "重庆网上房地产"
+ }
+ obj.m_date = new Date()
+ def query = new BasicDBObject("_id", obj._id)
+ LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
+
+ }
+ } else {
+ def udata = new BasicDBObject()
+ udata.put("city", d.city)
+ udata.put("region", newRegion)
+ udata.put("name", d.name)
+ udata.put("house_num", house_num)
+ if (d.unit_num) {
+ udata.put("unit_num", d.unit_num)
+ }
+ if (d.address) {
+ def address = AddressUtil.formatAddress(d.city, d.address)
+ if (address) {
+ udata.put("address", address)
+ }
+ }
+ udata.put("areas", [d.area])
+ udata.put("from", ["重庆网上房地产"])
+ udata.put("floors", maxHeight)
+ udata.put("c_date", new Date())
+ udata.put("status", 1)
+ LOUDONG_LINE_COLL.save(udata)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ def findMaxHeight(city, name,house_num, unit_num){
+ def q = new BasicDBObject()
+ q.put("city",city)
+ q.put("name",name)
+ q.put("house_num_desc",house_num)
+ q.put("unit_num",unit_num)
+ def dbcursor = loudongFdcConn.find(q)
+ def heights = []
+ for(def h : dbcursor){
+ if(h.nominal_floor && h.nominal_floor.isNumber()){
+ def height = h.nominal_floor.toInteger()
+ if(!heights.contains(height)){
+ heights << height
+ }
+ }
+ }
+ if(heights){
+ heights = heights.sort()
+ return heights[heights.size()-1]
+ }
+ return null
+ }
+
+ def findLoudongOnLine(city, region, name, house_num, unit_num){
+ def q = new BasicDBObject()
+ q.put("city",city)
+ q.put("region",region)
+ q.put("name",name)
+ q.put("house_num",house_num)
+ if(unit_num){
+ q.put("unit_num",unit_num)
+ }
+ def dbo = LOUDONG_LINE_COLL.findOne(q)
+ return dbo
+ }
+
+
def stat(){
def q = new BasicDBObject()
@@ -291,7 +684,51 @@ class CqfdcMatch {
}
def addLoudong(){
+// def stat = statBaseLoudong()
+// println stat
+// findAllLoundongByHouseNum();
+ def stat1 = statHouseLoudong()
+ println stat1
+// addHouseLoudong()
+ }
+ def addFdcLoudong(){
+// addFdcBaseLoudong()
+ addFdcHouseLoudong()
+ }
+
+ def statHouseLoudong(){
+ def q = new BasicDBObject()
+ q.put("city", "重庆")
+ def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q)
+ def total = dbcursor.size()
+ def ids = []
+ for(def d : dbcursor){
+ if(d.district_id && !ids.contains(d.district_id)){
+ ids << d.district_id
+ }
+ }
+ def map = [:]
+ map.rows_total = total
+ map.dist_count = ids.size()
+ return map
+ }
+
+ def statBaseLoudong(){
+ def q = new BasicDBObject()
+ q.put("city", "重庆")
+ def dbcursor = LOUDONG_LINE_COLL.find(q)
+ def total = dbcursor.size()
+ def ids = []
+ for(def d : dbcursor){
+ if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){
+ ids << d.id_in_seaweed
+ }
+ }
+ def map = [:]
+ map.rows_total = total
+ map.dist_count = ids.size()
+ return map
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy
new file mode 100644
index 000000000..6cf7730ee
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy
@@ -0,0 +1,28 @@
+package com.fangjia.script.temp.shenjian.didata
+
+import com.fangjia.script.util.ExcelUtils
+import com.fangjia.script.util.OpenApiClient
+
+/**
+ * 从excel中匹配出
+ */
+ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row ->
+ def city = row['A']
+ def region = row['B']
+ def district_name = row['C']
+
+ def gaodeRegion = row['H']
+ def gaodeName = row['I']
+
+ def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict")
+ if (jsonObject != null && jsonObject.precise != -1) {
+ row['D'] = jsonObject.get("name")
+ }
+
+ if (gaodeRegion && gaodeName) {
+ def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict")
+ if (jsonObject2 != null && jsonObject2.precise != -1) {
+ row['J'] = jsonObject2.get("name")
+ }
+ }
+});
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy
new file mode 100644
index 000000000..bdd0d1110
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy
@@ -0,0 +1,46 @@
+package com.fangjia.script.temp.shenjian.geo
+
+import com.fangjia.script.gj.location.Point
+import com.fangjia.script.util.MongoUtils
+import com.fangjia.script.xh.InsidePolygon
+import com.mongodb.BasicDBList
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+
+DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017)
+
+collection.find(new BasicDBObject()).forEach({ obj ->
+ if (obj.type == 'block') {
+ if (obj.region instanceof String) {
+ return;
+ }
+ BasicDBList reigon = (BasicDBList)obj.region
+ int count = 0
+ List info = new ArrayList<>()
+ for (String s : reigon) {
+ info.add(s)
+ }
+ if (info.size() > 1) {
+ boolean find = false
+ for (String s : info) {
+ if (find) {
+ break
+ }
+ collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 ->
+ String border = obj2.border
+ String points = border.replaceAll(",", "-").replaceAll(";", "-")
+ List pointList = InsidePolygon.createPoint(points);
+ com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat)
+ if (InsidePolygon.aboutZone(point, pointList)) {
+ collection.update(new BasicDBObject("_id", obj._id),
+ new BasicDBObject('$set', new BasicDBObject("region", obj2.name)));
+ find = true;
+ }
+ })
+ }
+ } else {
+ collection.update(new BasicDBObject("_id", obj._id),
+ new BasicDBObject('$set', new BasicDBObject("region", info.get(0))));
+ }
+ }
+})
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy
index bd49875b0..235d779e4 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy
@@ -2,6 +2,8 @@ package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.statistics.poi.AmapPoi
+import com.fangjia.script.statistics.poi.AmapPoiDetail
+import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
@@ -14,13 +16,15 @@ import java.util.concurrent.atomic.AtomicLong
class GetPoiFromExcel {
- static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi"
static final ExecutorService executorService = Executors.newFixedThreadPool(10)
+ static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
+ // 住宅类
+ static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0)
//List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
- List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false);
+ List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false);
for (File file : fileList) {
List> seaweeds = ExcelUtils.read(file.getPath())
for (List infoTemp : seaweeds) {
@@ -31,9 +35,10 @@ class GetPoiFromExcel {
String city = info.get(0)
String region = info.get(1)
String seaweedName = info.get(2)
- List pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302");
+ List pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
if (CollectionUtils.isNotEmpty(pois)) {
- String jsonStr = JSONObject.toJSONString(pois)
+ List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
+ String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
atomicLong.incrementAndGet()
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy
index 1a620af34..20def0a9b 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy
@@ -13,7 +13,13 @@ import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
+import com.mongodb.client.model.Filters
+import com.mongodb.client.model.UpdateOneModel
+import com.mongodb.client.model.UpdateOptions
+import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils
+import org.bson.Document
+import org.bson.conversions.Bson
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
@@ -27,74 +33,85 @@ import java.util.concurrent.Executors
*/
class SeaweedAroundPoiSingal {
- static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017)
+ static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) {
- def types = listAllType()
- //def types = "0601"
- DBObject query = new BasicDBObject()
- query.put("status", 1)
- conn136.find(query).batchSize(30).each {objTemp ->
- final def obj = objTemp
- def city = obj.city
- def lng = obj.lng
- def lat = obj.lat
- List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
- if (CollectionUtils.isNotEmpty(pois)) {
- String jsonStr = JSONObject.toJSONString(pois)
- HttpUtils.postForJson(URL, jsonStr)
- conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
- } else {
- conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
- }
- println("已处理小区:" + obj)
- }
+ getNewSeaweed()
+// def types = listAllType()
+// //def types = "0601"
+// DBObject query = new BasicDBObject()
+// query.put("status", 1)
+// conn136.find(query).batchSize(30).each {objTemp ->
+// final def obj = objTemp
+// def city = obj.city
+// def lng = obj.lng
+// def lat = obj.lat
+// List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
+// if (CollectionUtils.isNotEmpty(pois)) {
+// String jsonStr = JSONObject.toJSONString(pois)
+// HttpUtils.postForJson(URL, jsonStr)
+// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
+// } else {
+// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
+// }
+// println("已处理小区:" + obj)
+// }
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
- //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
+ filter.put("alias", 1)
+ filter.put("address", 1)
+ filter.put("addr_alias", 1)
filter.put("lng", 1)
filter.put("lat", 1)
- List list = []
+ List> writeModelList = new ArrayList<>()
+ def count = 0
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
- if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) {
- // 已存在小区,无需同步
- } else {
- DBObject newObj = new BasicDBObject()
- newObj.put("_id", obj._id)
- newObj.put("city", obj.city)
- newObj.put("region", obj.region)
- newObj.put("name", obj.name)
- try {
- newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng))
- newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat))
- } catch (Exception ex) {
- print("经纬度为空:" + obj._id + " " + obj)
- ex.printStackTrace()
- }
- newObj.put("status", 1)
- newObj.put("flag", "未获取周边POI")
+ Document updateBson = new Document()
+ Document update = new Document()
+ update.put("city", obj.city)
+ update.put("region", obj.region)
+ update.put("name", obj.name)
+ update.put("alias", obj.alias)
+ update.put("address", obj.address)
+ update.put("addr_alias", obj.addr_alias)
+ try {
+ update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
+ update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
+ } catch (Exception ex) {
+ print("经纬度为空:" + obj._id + " " + obj)
+ }
+ update.put("status", 1)
+ update.put("flag", "未精确获取高德信息")
+ updateBson.append('$set', update)
- list << newObj
- if (list.size() == 5000) {
- conn136.insert(list)
- list.clear()
- }
+ Bson filterBson = Filters.eq("_id", obj._id)
+ UpdateOptions updateOptions = new UpdateOptions();
+ updateOptions.upsert(true)
+ UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
+ writeModelList.add(updateOneModel)
+ if (writeModelList.size() == 5000) {
+ conn136.bulkWrite(writeModelList)
+ count += 5000
+ println(count)
+ writeModelList.clear()
}
}
- conn136.insert(list)
- list.clear()
+ if (writeModelList.size() > 0) {
+ conn136.bulkWrite(writeModelList)
+ writeModelList.clear()
+ }
}
static def getByIdAndInsert(String id) {
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy
new file mode 100644
index 000000000..ea6d9cc67
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy
@@ -0,0 +1,163 @@
+package com.fangjia.script.temp.shenjian.poi
+
+
+import com.alibaba.fastjson.JSONObject
+import com.fangjia.script.lixue.util.DBUtils
+import com.fangjia.script.statistics.poi.AmapPoi
+import com.fangjia.script.statistics.poi.AmapPoiDetail
+import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
+import com.fangjia.script.temp.shenjian.util.GaoDeUtils
+import com.fangjia.script.temp.shenjian.util.HttpUtils
+import com.fangjia.script.temp.shenjian.util.SeaweedUtils
+import com.fangjia.script.util.DbUtils
+import com.fangjia.script.util.MongoUtils
+import com.fangjia.script.wenyan.auto.KeywordSearchUtils
+import com.mongodb.BasicDBObject
+import com.mongodb.DBCollection
+import com.mongodb.DBObject
+import com.mongodb.client.model.Filters
+import com.mongodb.client.model.UpdateOneModel
+import com.mongodb.client.model.UpdateOptions
+import com.mongodb.client.model.WriteModel
+import org.apache.commons.collections.CollectionUtils
+import org.apache.commons.lang.StringUtils
+import org.bson.Document
+import org.bson.conversions.Bson
+import org.bson.types.ObjectId
+
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
+
+/**
+ * 小区POI信息精确查找
+ *
+ * @author shenjian
+ * @since 2022/5/25
+ */
+class SeaweedPoiDetail {
+
+ static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
+ static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
+ // 住宅类
+ static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
+ static final ExecutorService executorService = Executors.newFixedThreadPool(20)
+
+ static void main(String[] args) {
+ DBObject query = new BasicDBObject()
+ query.put("status", 1)
+ conn136.find(query).batchSize(1000).each {objTemp ->
+ final def obj = objTemp
+ executorService.submit(new Runnable() {
+ @Override
+ void run() {
+ def city = obj.city
+ def region = obj.region
+ def name = obj.name
+ def alias = obj.alias
+ def address = obj.address
+ def addr_alias = obj.addr_alias
+ StringJoiner stringJoiner = new StringJoiner("|")
+ // 主小区名搜索
+ stringJoiner.add(name)
+ // 根据别名搜
+ if (alias != null && CollectionUtils.isNotEmpty(alias)) {
+ for (String str : alias) {
+ stringJoiner.add(str)
+ }
+ }
+ // 根据路名搜
+ if (StringUtils.isNotBlank(address)) {
+ stringJoiner.add(address)
+ }
+ // 根据路别名搜
+ if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) {
+ for (String str : addr_alias) {
+ stringJoiner.add(str)
+ }
+ }
+
+ String keywords = stringJoiner.toString()
+ List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes)
+ if (CollectionUtils.isEmpty(pois)) {
+ pois = GaoDeUtils.searchPoi(city, name, region, typecodes)
+ }
+ if (CollectionUtils.isEmpty(pois)) {
+ pois = GaoDeUtils.searchPoi(city, address, region, typecodes)
+ }
+
+ List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
+// for (AmapPoi amapPoi : pois) {
+// List amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId());
+// if (CollectionUtils.isNotEmpty(amapPoiList)) {
+// detailList.addAll(amapPoiList)
+// }
+// }
+ if (CollectionUtils.isNotEmpty(detailList)) {
+ String jsonStr = JSONObject.toJSONString(detailList)
+ HttpUtils.postForJson(URL, jsonStr)
+ conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到")))
+ } else {
+ conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区")))
+ }
+ println("已处理小区:" + obj)
+ }
+ })
+ }
+ executorService.shutdown()
+ }
+
+ static getNewSeaweed() {
+ DBObject query = new BasicDBObject()
+ query.put("cat", "district")
+ query.put("visible", 0)
+
+ DBObject filter = new BasicDBObject()
+ filter.put("_id", 1)
+ filter.put("city", 1)
+ filter.put("region", 1)
+ filter.put("name", 1)
+ filter.put("alias", 1)
+ filter.put("address", 1)
+ filter.put("addr_alias", 1)
+ filter.put("lng", 1)
+ filter.put("lat", 1)
+
+ List> writeModelList = new ArrayList<>()
+ def count = 0
+ DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
+ Document updateBson = new Document()
+ Document update = new Document()
+ update.put("city", obj.city)
+ update.put("region", obj.region)
+ update.put("name", obj.name)
+ update.put("alias", obj.alias)
+ update.put("address", obj.address)
+ update.put("addr_alias", obj.addr_alias)
+ try {
+ update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
+ update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
+ } catch (Exception ex) {
+ print("经纬度为空:" + obj._id + " " + obj)
+ }
+ update.put("status", 1)
+ update.put("flag", "未精确获取高德信息")
+ updateBson.append('$set', update)
+
+ Bson filterBson = Filters.eq("_id", obj._id)
+ UpdateOptions updateOptions = new UpdateOptions();
+ updateOptions.upsert(true)
+ UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
+ writeModelList.add(updateOneModel)
+ if (writeModelList.size() == 5000) {
+ conn136.bulkWrite(writeModelList)
+ count += 5000
+ println(count)
+ writeModelList.clear()
+ }
+ }
+ if (writeModelList.size() > 0) {
+ conn136.bulkWrite(writeModelList)
+ writeModelList.clear()
+ }
+ }
+}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy
new file mode 100644
index 000000000..89dda4684
--- /dev/null
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy
@@ -0,0 +1,40 @@
+package com.fangjia.script.temp.shenjian.util;
+
+import com.alibaba.fastjson.JSON;
+import com.alibaba.fastjson.JSONArray;
+import com.alibaba.fastjson.JSONObject;
+
+/**
+ * 通用实体与DTO互相转换工具类
+ *
+ * @author shenjian
+ * @date 2021/06/16
+ */
+public class CommonDtoUtils {
+
+ public static T transformV(Class targetClass, Object source) {
+ if (source == null) {
+ return null;
+ } else {
+ try {
+ String jsonSource = JSON.toJSONString(source);
+ return JSONObject.parseObject(jsonSource, targetClass);
+ } catch (Exception var3) {
+ throw var3;
+ }
+ }
+ }
+
+ public static List transformListV(Class targetClass, List> listSource) {
+ if (listSource == null) {
+ return null;
+ } else {
+ try {
+ String jsonSource = JSON.toJSONString(listSource);
+ return JSONArray.parseArray(jsonSource, targetClass);
+ } catch (Exception var3) {
+ throw var3;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy
index 49c7c18de..6b880c5ea 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy
@@ -7,10 +7,12 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.statistics.poi.AmapPoi
+import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import org.apache.commons.lang.StringUtils
+import org.openqa.selenium.json.Json
import java.util.concurrent.atomic.AtomicInteger
@@ -88,12 +90,25 @@ class GaoDeUtils {
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
result.addAll(DataUtil.toMongoDbObj(json.pois))
- } else if (json && json.infocode && json.infocode == 10003) {
- throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封")
- } else if (json && json.infocode && json.infocode == 10004) {
+ } else if (json && json.infocode && json.infocode == "10044") {
+ synchronized (this) {
+ for (int i = 0; i < GAO_DE_AK.size(); i++) {
+ if (GAO_DE_AK[i] == params.key) {
+ if (i + 1 >= GAO_DE_AK.size()) {
+ System.exit(1)
+ }
+ keyUsed = new AtomicInteger(i+1)
+ }
+ }
+ break;
+ }
+ } else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
+ } else if (json && json.infocode && json.infocode == "10017") {
+ // 所请求的资源不可用
+ System.exit(1)
} else {
break
}
@@ -146,7 +161,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
- if (i + 1 > GAO_DE_AK.size()) {
+ if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -160,7 +175,9 @@ class GaoDeUtils {
break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
- System.exit(1)
+ //System.exit(1)
+ Thread.sleep(60000)
+ break;
} else {
break
}
@@ -222,12 +239,65 @@ class GaoDeUtils {
} catch(Exception ex) {
ex.printStackTrace()
}
+ } else if (json && json.infocode && json.infocode == "10044") {
+ synchronized (this) {
+ for (int i = 0; i < GAO_DE_AK.size(); i++) {
+ if (GAO_DE_AK[i] == params.key) {
+ if (i + 1 >= GAO_DE_AK.size()) {
+ System.exit(1)
+ }
+ keyUsed = new AtomicInteger(i+1)
+ }
+ }
+ }
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
- System.exit(1)
+ //System.exit(1)
+ Thread.sleep(60000)
+ }
+ return Collections.emptyList()
+ }
+
+ /**
+ * 获取详情
+ */
+ static List getAmapDetailById(id){
+ def params = [:]
+ params.key = GAO_DE_AK[keyUsed.intValue()]
+ params.id = id
+ params.output = 'json'
+ JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
+ if (json == null) {
+ json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
+ }
+ if (json && json.info && json.info == "OK") {
+ try {
+ List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
+ return ret
+ } catch(Exception ex) {
+ ex.printStackTrace()
+ }
+ } else if (json && json.infocode && json.infocode == "10044") {
+ synchronized (this) {
+ for (int i = 0; i < GAO_DE_AK.size(); i++) {
+ if (GAO_DE_AK[i] == params.key) {
+ if (i + 1 >= GAO_DE_AK.size()) {
+ System.exit(1)
+ }
+ keyUsed = new AtomicInteger(i+1)
+ }
+ }
+ }
+ } else if (json && json.infocode && json.infocode == "10004") {
+ // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
+ Thread.sleep(60000)
+ } else if (json && json.infocode && json.infocode == "10017") {
+ // 所请求的资源不可用
+ //System.exit(1)
+ Thread.sleep(60000)
}
return Collections.emptyList()
}
@@ -270,7 +340,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
- if (i + 1 > GAO_DE_AK.size()) {
+ if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -278,13 +348,13 @@ class GaoDeUtils {
}
break;
}
- } else if (json && json.infocode && json.infocode == "10004") {
+ } else if (json && json.infocode && (json.infocode == "10004" || json.infocode == "10021" || json.infocode == "10017")) {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
break;
- } else if (json && json.infocode && json.infocode == "10017") {
- // 所请求的资源不可用
- System.exit(1)
+ } else if (json && json.infocode && (json.infocode == "20000")) {
+ // 无效参数
+ break;
}
}
return result
@@ -326,7 +396,7 @@ class GaoDeUtils {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
- if (i + 1 > GAO_DE_AK.size()) {
+ if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
@@ -367,7 +437,16 @@ class GaoDeUtils {
return ["lng": arrs[0], "lat": arrs[1]]
}
} else if (json && json.infocode && json.infocode == "10044") {
- System.exit(1)
+ synchronized (this) {
+ for (int i = 0; i < GAO_DE_AK.size(); i++) {
+ if (GAO_DE_AK[i] == params.key) {
+ if (i + 1 >= GAO_DE_AK.size()) {
+ System.exit(1)
+ }
+ keyUsed = new AtomicInteger(i+1)
+ }
+ }
+ }
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy
index 668172385..c17285a78 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy
@@ -128,10 +128,10 @@ class SeaweedUtils {
*/
static formatHouseholdCount(def household_count) {
if (household_count && FieldValidatorUtils.checkValue(household_count) ) {
- if (Long.valueOf(household_count) < 50) {
+ if (Integer.valueOf(household_count) < 50) {
return "" // 户数
}
- return household_count
+ return household_count.toInteger()
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
index 1d80f9b2a..9a9fd90ad 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy
@@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{
def filterSpacialChar(def s){
def pattern = ~/[()!;\·]\[【】()]/
def matcher = pattern.matcher(s)
+
def count = matcher.getCount()
if(count && count>0){
return matcher.replaceAll('_')
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy
index 97af5b6dc..de2db639a 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy
@@ -43,8 +43,8 @@ class DataStat {
def source = "58同城"//
def citys = findCrawlCity(source, 202202);
for(def city : citys){
- def count1 = findCount(source, city, 202202);
- def count2 = findCount(source, city, 202203);
+ def count1 = findCount(source, city, 202203);
+ def count2 = findCount(source, city, 202204);
println city +" "+count1+" "+count2
// println count1
// println "--- ----"
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy
index ee0977a28..ed1b90eba 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy
@@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = []
static def totalCount = new AtomicLong()
- def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
+ def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
"保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872",
"沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556",
"石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606",
"绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572",
"淮北":"huaibei|9357","滁州":"chuzhou|10266",
- "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
+ "蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
"兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362",
"芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222",
"南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325",
@@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) {
//addContent("福州","https://fz.58.com/xiaoqu/", 70);
- def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"]
+// "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州",
+ def citys = ["唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","北京","衡阳","南京","滨州","张家口","商丘","赣州","湖州","安庆","聊城","枣庄","阜阳","西安","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳"]
for(def city : citys){
def str = cityList.get(city)
for(def i=0;i<1;i++){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
index fec88b8c5..b620f3098 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房",
- def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"]
+ def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -26,7 +26,7 @@ class SynchDistrictPrice {
def run(){
- def citys = CommonUtils.findAllCity()
+ def citys = ["重庆"]//CommonUtils.findAllCity()
sourceMap.each { k, v ->
for(def city : citys){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
index bd0fbb3f3..4866f25b8 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){
- def count1 = findTotalCount(source, 202202);
- def count2 = findTotalCount(source, 202203);
+ def count1 = findTotalCount(source, 202203);
+ def count2 = findTotalCount(source, 202204);
println source +" "+count1 +" "+count2
}
}
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy
index fe433f087..21887e3db 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy
@@ -169,7 +169,7 @@ public class CommonUtils {
static void sendMessage(String queueName, byte[] bytes){
ConnectionFactory factory = new ConnectionFactory();
- factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f");
+ factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f");
Connection conn = factory.newConnection();
Channel channel = conn.createChannel();
channel.queueDeclare(queueName, true, false, false, null);
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy
index 69a75d454..7e6e73263 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy
@@ -92,11 +92,6 @@ public class DbUtils{
}
}
-
- public static void executeQueryOnSeawater(BasicDBObject query,Closure c, BasicDBObject fields = null){
- seawater.find(query,fields).batchSize(10000).each(c)
- }
-
public static void executeQueryOnSeaweed(BasicDBObject query,Closure c, BasicDBObject fields = null){
SEAWEED_SLAVE.find(query,fields).batchSize(1000).each(c)
}
@@ -171,10 +166,6 @@ public class DbUtils{
return SEAWEED_SLAVE.findOne(query,fields);
}
- public static int executeCountOnSeawater(BasicDBObject query){
- return seawater.count(query);
- }
-
public static BasicDBObject buildSeaweedQuery(def city = null, def region = null, def name = null, def cat = "district", def visible = 0){
BasicDBObject query = new BasicDBObject()
if(city){
@@ -387,31 +378,6 @@ public class DbUtils{
return district.lat && district.lng && district.lng > 0 && district.lat > 0
}
-/**
- * 记录修改小区库的操作日志
- * @param city
- * @param region
- * @param name
- * @return
- */
- def static saveDeleteLog(String city, String region,String name,String category,int status){
- if(status!=1){
- return
- }
- DBObject obj = new BasicDBObject()
- obj.put("city", city);
- obj.put("region", region);
- obj.put("district_name", name);
- obj.put("category" , category)
- obj.put("origin" , "脚本")
- if(status==1){
- obj.put("type" , "remove")
- }
- obj.put("c_date" , new Date())
- obj.put("m_date" , new Date())
- OPERATION_LOG.save(obj)
- }
-
public static DBCollection getCollection(String host, int port, String dbName ,String collName){
def collection = COLL_CACHE["${host}#${port}#${dbName}#${collName}"]
if(!collection){
diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy
index 5ef52da25..3d4e855f2 100644
--- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy
+++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy
@@ -229,6 +229,153 @@ public class OpenApiClient {
return null
}
+ public static JSONObject searchLoudong(String id, String houseNum, String unitNum){
+ try{
+ Map params = new HashMap();
+ params.put("id", id);
+ params.put("houseNumber", houseNum);
+ params.put("unitNumber", unitNum);
+ params.put("token", TOKEN);
+ String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ while (result == null) {
+ result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
+ println("${id} ${houseNum} 程序调用异常")
+ Thread.sleep(6000)
+ }
+ if(result){
+ if (result.getInt("code") == 200) {
+ return result;
+ } else if (result.getInt("code") !=0) {
+ println("${id} ${houseNum}")
+ }
+ }
+
+ } catch (Exception e) {
+ println("${id} ${houseNum} ${e.getMessage()}")
+ e.printStackTrace();
+ }
+ return null
+ }
+
+ public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){
+ try{
+ Map params = new HashMap();
+ params.put("id", id);
+ params.put("houseNumber", houseNum);
+ params.put("unitNumber", unitNum);
+ params.put("roomNumber", roomNum);
+ params.put("token", TOKEN);
+ String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ while (result == null) {
+ result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
+ println("${id} ${houseNum} 程序调用异常")
+ Thread.sleep(6000)
+ }
+ if(result){
+ if (result.getInt("code") == 200) {
+ return result;
+ } else if (result.getInt("code") !=0) {
+ println("${id} ${houseNum}")
+ }
+ }
+
+ } catch (Exception e) {
+ println("${id} ${houseNum} ${e.getMessage()}")
+ e.printStackTrace();
+ }
+ return null
+ }
+
+
+ public static JSONObject webankEvaluate(String city, String region, String address, String area){
+ try{
+ Map params = new HashMap();
+ params.put("city", city);
+ params.put("district", region);
+ params.put("address", address);
+ params.put("area", area.toString());
+ params.put("token", TOKEN);
+ String content = new HttpRequest().get("http://open.fangjia.com/webank/evaluate", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ while (result == null) {
+ result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/webank/evaluate", params, "utf-8")
+ println("${city} ${address} 程序调用异常")
+ Thread.sleep(6000)
+ }
+ if(result){
+ if (result.getInt("code") == 200) {
+ return result;
+ } else if (result.getInt("code") !=0) {
+ println("${city} ${address} 程序调用异常")
+ }
+ }
+
+ } catch (Exception e) {
+ println("${city} ${address} ${e.getMessage()}")
+ e.printStackTrace();
+ }
+ return null
+ }
+
+
+ public static JSONObject standard2Action(String city, String address) {
+ try {
+ Map params = new HashMap();
+ params.put("city", city);
+ params.put("address", address);
+ params.put("token", TOKEN);
+ params.put("__refresh_cache","1")
+
+ String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ while (result == null) {
+ result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8")
+ println("${city} ${address} 程序调用异常")
+ Thread.sleep(6000)
+ }
+ if (result.getInt("code") == 0) {
+ return result.getJSONObject("result");
+ } else if (result.getInt("code") !=0) {
+ println("${city} ${address}")
+ }
+ } catch (Exception e) {
+ println("${city} ${address} ${e.getMessage()}")
+ e.printStackTrace();
+ }
+ return null
+ }
+
+
+ public static JSONObject standard1Action(String city, String address) {
+ try {
+ Map params = new HashMap();
+ params.put("city", city);
+ params.put("address", address);
+ params.put("token", TOKEN);
+ params.put("__refresh_cache","1")
+
+ String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ while (result == null) {
+ result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8")
+ println("${city} ${address} 程序调用异常")
+ Thread.sleep(6000)
+ }
+ if (result.getInt("code") == 200) {
+ return result.getJSONObject("result");
+ } else if (result.getInt("code") !=0) {
+ println("${city} ${address}")
+ }
+ } catch (Exception e) {
+ println("${city} ${address} ${e.getMessage()}")
+ e.printStackTrace();
+ }
+ return null
+ }
+
+
public static JSONObject standardize(String city, String address) {
try {
Map params = new HashMap();
@@ -593,6 +740,91 @@ public class OpenApiClient {
}
+ static JSONObject facilitys(String city, String region, String name) {
+ Map params = new HashMap();
+ params.put("token", TOKEN);
+ params.put("city", city)
+ params.put("region", region)
+ params.put("name", name)
+ try {
+ String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+
+ return result
+ } catch (Exception e) {
+
+ }
+ return null
+ }
+
+ static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) {
+ Map params = new HashMap();
+ params.put("token", TOKEN);
+ params.put("city", city)
+ params.put("district", region)
+ params.put("name", name)
+ params.put("startDate", startDate)
+ params.put("limit", "100")
+ try {
+ String content = new HttpRequest().get("http://open.fangjia.com/auction/searchPaimais", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ return result
+ } catch (Exception e) {
+
+ }
+ return null
+ }
+
+
+ static JSONObject searchDealPrice(String id, String startDate, limit) {
+ Map params = new HashMap();
+ params.put("token", TOKEN);
+ params.put("id", id)
+ params.put("startDate", startDate)
+ params.put("limit", limit.toString())
+ try {
+ String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ return result
+ } catch (Exception e) {
+
+ }
+ return null
+ }
+
+ static JSONObject houseList(String id, String startDate, limit) {
+ Map params = new HashMap();
+ params.put("token", TOKEN);
+ params.put("id", id)
+ params.put("startTime", startDate)
+ params.put("pageSize", limit.toString())
+ try {
+ String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ return result
+ } catch (Exception e) {
+
+ }
+ return null
+ }
+
+
+ static JSONObject evaluaionList(String id, String startDate, limit) {
+ Map params = new HashMap();
+ params.put("token", TOKEN);
+ params.put("id", id)
+ params.put("startDate", startDate)
+ params.put("limit", limit.toString())
+ try {
+ String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8")
+ JSONObject result = JSONObject.fromObject(content);
+ return result
+ } catch (Exception e) {
+
+ }
+ return null
+ }
+
static JSONObject districtDetail(String city, String region, String name) {
Map params = new HashMap();
params.put("token", TOKEN);
diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java
new file mode 100644
index 000000000..8192b359c
--- /dev/null
+++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java
@@ -0,0 +1,71 @@
+package com.fangjia.script.statistics.poi;
+
+import java.util.List;
+
+/**
+ * @author shenjian
+ * @since 2022/5/25
+ */
+@lombok.NoArgsConstructor
+@lombok.Data
+public class AmapPoiDetail {
+
+ @com.alibaba.fastjson.annotation.JSONField(name = "id")
+ private String id;
+ @com.alibaba.fastjson.annotation.JSONField(name = "parent")
+ private List> parent;
+ @com.alibaba.fastjson.annotation.JSONField(name = "childtype")
+ private List> childtype;
+ @com.alibaba.fastjson.annotation.JSONField(name = "name")
+ private String name;
+ @com.alibaba.fastjson.annotation.JSONField(name = "type")
+ private String type;
+ @com.alibaba.fastjson.annotation.JSONField(name = "typecode")
+ private String typecode;
+ @com.alibaba.fastjson.annotation.JSONField(name = "biz_type")
+ private List> bizType;
+ @com.alibaba.fastjson.annotation.JSONField(name = "address")
+ private String address;
+ @com.alibaba.fastjson.annotation.JSONField(name = "location")
+ private String location;
+ @com.alibaba.fastjson.annotation.JSONField(name = "tel")
+ private String tel;
+ @com.alibaba.fastjson.annotation.JSONField(name = "pname")
+ private String pname;
+ @com.alibaba.fastjson.annotation.JSONField(name = "cityname")
+ private String cityname;
+ @com.alibaba.fastjson.annotation.JSONField(name = "adname")
+ private String adname;
+ @com.alibaba.fastjson.annotation.JSONField(name = "importance")
+ private List> importance;
+ @com.alibaba.fastjson.annotation.JSONField(name = "shopid")
+ private List> shopid;
+ @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo")
+ private String shopinfo;
+ @com.alibaba.fastjson.annotation.JSONField(name = "poiweight")
+ private List> poiweight;
+ @com.alibaba.fastjson.annotation.JSONField(name = "distance")
+ private String distance;
+ @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext")
+ private BizExtDTO bizExt;
+ @com.alibaba.fastjson.annotation.JSONField(name = "photos")
+ private List photos;
+
+ @lombok.NoArgsConstructor
+ @lombok.Data
+ public static class BizExtDTO {
+ @com.alibaba.fastjson.annotation.JSONField(name = "rating")
+ private List> rating;
+ @com.alibaba.fastjson.annotation.JSONField(name = "cost")
+ private String cost;
+ }
+
+ @lombok.NoArgsConstructor
+ @lombok.Data
+ public static class PhotosDTO {
+ @com.alibaba.fastjson.annotation.JSONField(name = "title")
+ private List> title;
+ @com.alibaba.fastjson.annotation.JSONField(name = "url")
+ private String url;
+ }
+}
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java
index 46545fc04..15909a4c9 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java
@@ -1,12 +1,12 @@
package com.fangjia.service.address;
import com.fangjia.core.RegionData;
-import com.fangjia.core.util.*;
+import com.fangjia.core.util.DataUtil;
+import com.fangjia.core.util.GeoUtils;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
-import com.fangjia.dal.mongo.model.AmapPoi;
import com.fangjia.dal.mongo.model.bkoff.HouseLoudong;
import com.fangjia.dal.mongo.model.bkoff.Loudong;
import com.fangjia.dal.seaweed.dao.SeaweedDao;
@@ -16,17 +16,15 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors;
import com.fangjia.service.address.processor.AddressProcessors;
import com.fangjia.service.api.AliyunService;
import com.fangjia.service.bo.AliyunBo;
-import com.fangjia.service.bo.MatchAddressScore;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.mongo.addrepo.AddressRepoService;
import com.fangjia.service.mongo.bkoff.HouseLoudongService;
import com.fangjia.service.mongo.bkoff.LoudongService;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
-import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
-import org.bson.types.ObjectId;
+
import java.util.*;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
@@ -759,6 +757,13 @@ public class AddressParserServiceImpl implements AddressParserService {
double simScore;
boolean onSameRoad = false;
boolean sameDistrict = isSameDistrict(address, metaData);
+ if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){
+ return 0;
+ }
+// //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区
+// if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){
+// return 0;
+// }
Iterator> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator();
Map.Entry addrEntry;
while(addrEntrySet.hasNext()){
@@ -808,7 +813,6 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
}
-
String keywordPinYin = address.getKeywordsPinYin();
String keywords = address.getKeywords();
String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin();
@@ -892,7 +896,7 @@ public class AddressParserServiceImpl implements AddressParserService {
if(name.length() == 2 && (name.equals(address.getOrgAddress()) || name.equals(address.getEffectiveAddress()))){
keywordScore = Math.max(keywordScore, namePriority);
}else if(name.length() > 3 && (address.getOrgAddrePinYin().contains(namePinYin) || enhanceContains(address.getOrgAddress(), name))){
- double similar = AddressParserUtil.getSimilarityRatio(name, address.getEffectiveAddress());
+ double similar = AddressParserUtil.getSimilarityRatio(name, address.getOrgAddress());
if(similar == 1){
keywordScore = Math.max(similar, keywordScore) * namePriority * 1.0001;
}else if(similar > 0.1){
@@ -907,20 +911,27 @@ public class AddressParserServiceImpl implements AddressParserService {
if (standardAddressScore > 0 && keywordScore > 0) {
matchLevelScore = 0.9;
similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE;
- } else if(keywordScore > 0 && streetScore > 0){
- matchLevelScore = 0.8;
- similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE;
} else if (standardAddressScore > 0) {
- matchLevelScore = 0.7;
+ matchLevelScore = 0.8;
similarityScore = standardAddressScore * SIMILARITY_RATE;
+ } else if(keywordScore > 0 && streetScore > 0){
+ matchLevelScore = 0.7;
+ similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE;
} else if (keywordScore > 0) {
matchLevelScore = 0.6;
similarityScore = keywordScore * SIMILARITY_RATE;
} else if (fuzzyKeywordsScore > 0) {
- matchLevelScore = 0.5;
+ if(address.hasCoordinate() && metaData.getLng() != null && GeoUtils.calculateDistance(address.getLng(), address.getLat(),
+ metaData.getLng(), metaData.getLat()) < 100){
+ matchLevelScore = 0.7;
+ }else{
+ matchLevelScore = 0.4;
+ }
similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE;
+ }else if(metaData.getId().equals(address.getPolygonSeaweedId())){
+ matchLevelScore = 0.4;
+ similarityScore = 0.05;
}
-
if (similarityScore > 0) {
if(metaData.getBuildingType() != null){
if ("shop".equals(metaData.getBuildingType().toString())) {
@@ -942,10 +953,6 @@ public class AddressParserServiceImpl implements AddressParserService {
* similarityScore = 相似度部分(0.07) + 类型部分(0.01) < 0.1
* */
double score = matchLevelScore + similarityScore;
- if(address.hasCoordinate() && metaData.hasCoordinate()
- && GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) > 3 * 1000){
- score = 0d;
- }
return score;
}
@@ -980,7 +987,7 @@ public class AddressParserServiceImpl implements AddressParserService {
return false;
}
if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){
- return false;
+ return true;
}
String district = metaData.getDistrict();
return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict());
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java
index 720da51bb..ed723c1d5 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java
@@ -61,6 +61,8 @@ public class Address {
private String suggestDistrict;
private String suggestName;
private String suggestNamePinYin;
+ private String polygonSeaweedId;
+ private String geoRegion;
private List buildingNumbers;
private PreciseLevel preciseLevel;
private boolean pinYin;
@@ -750,6 +752,14 @@ public class Address {
return false;
}
+ public String getGeoRegion() {
+ return geoRegion;
+ }
+
+ public void setGeoRegion(String geoRegion) {
+ this.geoRegion = geoRegion;
+ }
+
@Override
public String toString() {
return "Address{" +
@@ -809,6 +819,14 @@ public class Address {
'}';
}
+ public String getPolygonSeaweedId() {
+ return polygonSeaweedId;
+ }
+
+ public void setPolygonSeaweedId(String polygonSeaweedId) {
+ this.polygonSeaweedId = polygonSeaweedId;
+ }
+
public String toMd5Hex(){
String keyProperties = "Address{" +
"city='" + city + '\'' +
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java
index aa72e6d01..d1a15d156 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java
@@ -20,18 +20,16 @@ public class AddressProcessors{
static {
//修改顺序需要调整process i 的值
- addressProcessors.add(new TokenProcessor());
addressProcessors.add(new ProximalCharactersProcessor());
addressProcessors.add(new PreNoiseProcessor());
addressProcessors.add(new AddressFormatProcessor());
- addressProcessors.add(new TokenProcessor());
addressProcessors.add(new FloorProcessor());
addressProcessors.add(new TooltipProcessor());
addressProcessors.add(new ProvinceProcessor());
addressProcessors.add(new CityProcessor());
addressProcessors.add(new DistrictProcessor());
- addressProcessors.add(new LoopLineProcessor());
addressProcessors.add(new TownshipProcessor());
+ addressProcessors.add(new LoopLineProcessor());
addressProcessors.add(new StageAndPartitionProcessor());
addressProcessors.add(new DirectionProcessor());
addressProcessors.add(new OffsetProcessor());
@@ -44,7 +42,7 @@ public class AddressProcessors{
addressProcessors.add(new NumericProcessor());
addressProcessors.add(new ProvinceAdjustProcessor());
addressProcessors.add(new FormatProcessor());
-// addressProcessors.add(new GeoProcessor());
+ addressProcessors.add(new GeoProcessor());
}
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java
index 6f88c20ef..fce876ec2 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java
@@ -52,8 +52,9 @@ public class CityProcessor extends AbstractAddressProcessor {
}
if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){
String formatCity = RegionData.formatCity(city);
- String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null;
- String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity;
+ address.setCity(formatCity);
+ String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null;
+ String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity;
/**如果省份和城市信息相互矛盾 则用区域推断城市*/
if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){
List possibleCities = speculateCityNameByRegion(address, city);
@@ -191,15 +192,12 @@ public class CityProcessor extends AbstractAddressProcessor {
}else{
weight.score += 2;
}
-
}else if(address.contains(city)){
index = address.indexOf(city);
weight.index = index;
index += city.length();
if(province != null && (orgAddress.contains(province + city) || orgAddress.contains(ProvinceData.getDisplay(province) + city))){
weight.score += 2;
- }else{
- weight.score += 1;
}
}
Map> regionMap = RegionData.getAllRegions(city);
@@ -208,9 +206,11 @@ public class CityProcessor extends AbstractAddressProcessor {
if(matchResult !=null){
address = address.replace(matchResult.matchContent, "");
}
- if(address.indexOf(region, index) > -1){
+ if(address.indexOf(region + "区", index) > -1 || address.indexOf(region + "县", index) > -1){
if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){
weight.score += 2;
+ }else if(city.equals(addr.getCity())){
+ weight.score += 10;
}else{
weight.score += 1;
}
@@ -221,9 +221,13 @@ public class CityProcessor extends AbstractAddressProcessor {
List alias = regionMap.get(region);
if(alias != null){
for(String alia : alias){
- if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1
+ if(StringUtils.isNotBlank(alia) && !alia.equals(region) && alia.length() > 2 && address.indexOf(alia, index) > -1
&& (alia.endsWith("市")|(alia.endsWith("县")))){
- weight.score += 1;
+ if(city.equals(addr.getCity())){
+ weight.score += 10;
+ }else{
+ weight.score += 1;
+ }
weight.index = address.indexOf(alia, index);
break;
}
@@ -237,16 +241,6 @@ public class CityProcessor extends AbstractAddressProcessor {
}
List weights = new ArrayList(weightMap.values());
if(weights.size() > 0){
- /*String street = getStreet(address);
- if(street != null){
- Iterator iterator = weights.iterator();
- while(iterator.hasNext()){
- FragmentWeight fragmentWeight = iterator.next();
- if(match(street, ".*" + fragmentWeight.text + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)")){
- iterator.remove();
- }
- }
- }*/
Collections.sort(weights);
}
if(weights.size() > 0){
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java
index 5cfba96b5..697e69f8d 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java
@@ -167,6 +167,9 @@ public class DistrictProcessor extends AbstractAddressProcessor {
address.setDevelopmentZone(dz);
}
}
+ if(address.getSuggestDistrict() != null && address.getDistrict() == null){
+ address.setDistrict(address.getSuggestDistrict());
+ }
address.setAddress(orgAddress);
address.setEffectiveAddress(address.getAddress());
return RepeatStatus.NEXT;
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java
index 83d74a0d4..e3ef9a3da 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java
@@ -5,8 +5,12 @@ import com.fangjia.core.util.HttpRequest;
import com.fangjia.service.address.mode.Address;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
-import org.apache.commons.lang.StringUtils;
-import java.util.*;
+import org.apache.log4j.Logger;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* 地址经纬度处理器
@@ -16,78 +20,69 @@ import java.util.*;
public class GeoProcessor extends AbstractAddressProcessor {
private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde";
-
- List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口");
- List DISTRICT_LEVEL = Arrays.asList("区县", "开发区", "乡镇", "村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口");
+ private static final Logger logger = Logger.getLogger(GeoProcessor.class);
+ List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口", "门牌号");
@Override
public RepeatStatus process(Address address) {
-
if (address.hasCoordinate()) {
return RepeatStatus.NEXT;
}
- List queryTypes = new ArrayList();
- List addressList = new ArrayList();
- if (address.isStandardAddress()) {
- addressList.add(address.getStandardAddress());
- queryTypes.add("StandardAddress");
- }
- if (StringUtils.isNotBlank(address.getOrgKeywords())) {
- addressList.add(address.getOrgKeywords());
- queryTypes.add("Keywords");
+ String addr = address.getEffectiveAddress();
+ if(addr == null || addr.length() < 3){
+ addr = address.getOrgAddress();
}
HttpRequest request = new HttpRequest();
Map params = new HashMap();
- params.put("address", StringUtils.join(addressList, "|"));
+ params.put("address", addr);
params.put("key", KEY);
params.put("city", address.getCity());
- params.put("batch", "true");
params.put("output", "json");
try {
JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8");
+ logger.info("高德API:" + response.toString());
if (response.getInt("status") == 1) {
JSONArray geocodes = response.getJSONArray("geocodes");
- if(queryTypes.contains("StandardAddress")){
- JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("StandardAddress"));
- String level = geocode.getString("level");
- if(DISTRICT_LEVEL.contains(level)){
- if(StringUtils.isBlank(address.getDistrict())){
- address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district")));
+ if(geocodes.size() > 0){
+ JSONObject json = geocodes.getJSONObject(0);
+ if(json.get("district") instanceof String){
+ String city = address.getCity();
+ if(city == null){
+ city = RegionData.formatCity(json.getString("city"));
}
+ String region = RegionData.formatRegion(city, json.getString("district"));
+ address.setGeoRegion(region);
}
- if(geocode.containsKey("street") && geocode.containsKey("number")){
- String geocodeAddress = geocode.getString("street") + geocode.getString("number");
- if(address.getStandardAddress().contains(geocodeAddress)){
- String location = geocode.getString("location");
- address.setLng(Double.parseDouble(location.split(",")[0]));
- address.setLat(Double.parseDouble(location.split(",")[1]));
- }
- }
- }
- if(queryTypes.contains("Keywords")){
- JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("Keywords"));
- String level = geocode.getString("level");
- if(DISTRICT_LEVEL.contains(level)){
- if(StringUtils.isBlank(address.getDistrict())){
- address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district")));
- }
- }
- if(POI_LEVELS.contains(level)){
- String formattedAddress = geocode.getString("formatted_address");
- formattedAddress = formattedAddress
- .replaceFirst(geocode.getString("province"), "")
- .replaceFirst(geocode.getString("city"), "")
- .replaceFirst(geocode.getString("district"), "");
- if(formattedAddress.equals(address.getOrgKeywords())){
- String location = geocode.getString("location");
- address.setLng(Double.parseDouble(location.split(",")[0]));
- address.setLat(Double.parseDouble(location.split(",")[1]));
+ if(POI_LEVELS.contains(json.getString("level"))){
+ //对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼
+ if("门牌号".equals(json.getString("level"))){
+ if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){
+ return RepeatStatus.NEXT;
+ }
}
+ String location = json.getString("location");
+ address.setLng(Double.valueOf(location.split(",")[0]));
+ address.setLat(Double.valueOf(location.split(",")[1]));
+ String tmp = "{ \"_source\": [\"seaweed_id\"], \"query\": {\"geo_shape\": {\"shape\": {\"shape\": {\"type\": \"Point\",\"coordinates\":[location]},\"relation\": \"CONTAINS\"}}}}";
+ tmp = tmp.replace("location", location);
+ Map headers = new HashMap();
+ headers.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz");
+ JSONObject resp = JSONObject.fromObject(request.postWithBody("http://192.168.0.206:9200/seaweed_polygon/_search", JSONObject.fromObject(tmp),headers,"UTF-8"));
+ if(resp.containsKey("hits") && resp.getJSONObject("hits").containsKey("hits")){
+ JSONArray hits = resp.getJSONObject("hits").getJSONArray("hits");
+ if(hits.size() > 0){
+ String seaweedId = hits.getJSONObject(0).getJSONObject("_source").getString("seaweed_id");
+ address.setPolygonSeaweedId(seaweedId);
+ }
+ }
+
}
}
+
}
} catch (Exception e) {
-
+ logger.error(e);
+ e.printStackTrace();
}
return RepeatStatus.NEXT;
}
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java
index 547d8d2cb..b982d0b4f 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java
@@ -22,7 +22,7 @@ public class KeywordProcessor extends AbstractAddressProcessor {
static final List KEYWORDS_PATTERNS = Arrays.asList(Pattern.compile(".*[^一二三四五六七八九十\\d]{3,}号"));
static final List KEYWORDS = Arrays.asList("居委会", "辖区", "街道办", "街道", "办事处", "村委", "管理区");
static final List NOISE_PATTERNS = Arrays.asList(Pattern.compile("^.*\\d+队"), Pattern.compile("^\\d+号院$"));
- static final List INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属院"));
+ static final List INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属[区院]"), Pattern.compile("宿舍"), Pattern.compile("生活区"), Pattern.compile("公寓"));
static final Pattern INVALID_PATTERN = Pattern.compile("^([-\\\\、])+$");
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java
index 4d3b65b14..3921fddf5 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java
@@ -42,12 +42,10 @@ public class TooltipProcessor extends AbstractAddressProcessor {
if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){
address.setTooltip(tooltip);
address.setStreet(getStreet(tooltip));
- } else if(containsDistrictName(tooltip)){
- address.setTooltip(tooltip);
- } else if(RegionData.isRegion(address.getCity(), tooltip)){
+ }else if(RegionData.isRegion(address.getCity(), tooltip)){
address.setSuggestDistrict(tooltip);
}else{
- address.addNosie(tooltip);
+ address.setTooltip(tooltip);
}
}
}
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java
index 5236b7d55..797d9e367 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java
@@ -15,15 +15,6 @@ public class VillageNameProcessor extends AbstractAddressProcessor {
@Override
public AbstractAddressProcessor.RepeatStatus process(Address address) {
String orgAddress = address.getAddress();
- if(address.getTokens()!=null){
- for(Token token:address.getTokens()){
- if("district".equals(token.getKeywordTypes())){
- address.setKeywords(token.getName());
- address.setAddress(orgAddress.replace(token.getName(),""));
- return RepeatStatus.NEXT;
- }
- }
- }
List names = tokenDistrictNames(orgAddress);
List kws = new ArrayList();
for(String name : names){
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java
index de010e201..c216a1eba 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java
@@ -1,24 +1,16 @@
package com.fangjia.service.cnAnalyzer;
-import java.io.IOException;
-import java.io.StringReader;
-import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.solr.client.solrj.util.ClientUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import com.fangjia.core.util.DataUtil;
import com.goojia.solr.ganalytics.IKSegmentation;
import com.goojia.solr.ganalytics.Lexeme;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.io.UnsupportedEncodingException;
+import java.util.*;
/**
* 房价网分词器
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java
index 3965c9f00..c38f74a88 100644
--- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java
@@ -240,7 +240,7 @@ public class AddressRepoServiceImpl implements AddressRepoService {
}
private boolean isAvailablePoi(AddressMatchMetaData data){
- if(data.getType() != null && data.getType().contains("路")){
+ if(data.getType() != null && (data.getType().contains("路") || data.getType().contains("街道"))){
return false;
}
return true;
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java
new file mode 100644
index 000000000..bc8c159de
--- /dev/null
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java
@@ -0,0 +1,9 @@
+package com.fangjia.service.open;
+
+import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
+
+public interface CommunityBasePriceService {
+
+ CommunityBasePrice getById(String id);
+}
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java
new file mode 100644
index 000000000..c0eb458f1
--- /dev/null
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java
@@ -0,0 +1,8 @@
+package com.fangjia.service.open;
+
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
+
+public interface CommunityPoiService {
+
+ CommunityPoi getById(String id);
+}
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java
new file mode 100644
index 000000000..049e5ae16
--- /dev/null
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java
@@ -0,0 +1,22 @@
+package com.fangjia.service.open.impl;
+
+import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
+import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
+import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
+import com.fangjia.service.open.CommunityBasePriceService;
+import com.fangjia.service.open.CommunityPoiService;
+
+public class CommunityBasePriceServiceImpl implements CommunityBasePriceService {
+
+ private CommunityBasePriceDao communityBasePriceDao;
+
+ @Override
+ public CommunityBasePrice getById(String id) {
+ return communityBasePriceDao.findById(id);
+ }
+
+ public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) {
+ this.communityBasePriceDao = communityBasePriceDao;
+ }
+}
diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java
new file mode 100644
index 000000000..8ea0b05e1
--- /dev/null
+++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java
@@ -0,0 +1,19 @@
+package com.fangjia.service.open.impl;
+
+import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
+import com.fangjia.dal.mongo.open.model.CommunityPoi;
+import com.fangjia.service.open.CommunityPoiService;
+
+public class CommunityPoiServiceImpl implements CommunityPoiService {
+
+ private CommunityPoiDao communityPoiDao;
+
+ @Override
+ public CommunityPoi getById(String id) {
+ return communityPoiDao.findById(id);
+ }
+
+ public void setCommunityPoiDao(CommunityPoiDao communityPoiDao) {
+ this.communityPoiDao = communityPoiDao;
+ }
+}
diff --git a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css
index 0b498e441..805853bc9 100644
--- a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css
+++ b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css
@@ -20,7 +20,7 @@
.officeMap{width:100%;height:400px;}
.main_content.partner{overflow:hidden;}
-.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:24px;margin-bottom:25px;cursor:pointer;}
+.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:15px;margin-bottom:25px;cursor:pointer;}
.partner a.partner11{background-position:0 0;}
.partner a.partner12{background-position:-204px 0;}