Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_bohai

This commit is contained in:
2022-06-15 15:38:39 +08:00
95 changed files with 7887 additions and 708 deletions
@@ -158,37 +158,36 @@
<a class="partner13"></a> <a class="partner13"></a>
<#-- <a class="partner14"></a>--> <#-- <a class="partner14"></a>-->
<a class="partner15"></a> <a class="partner15"></a>
<a class="partner16" style="margin-right:0;"></a> <a class="partner16"></a>
<a class="partner21"></a> <a class="partner21"></a>
<a class="partner22"></a> <a class="partner22"></a>
<a class="partner23"></a> <a class="partner23"></a>
<a class="partner24"></a> <a class="partner24"></a>
<a class="partner25"></a> <a class="partner25"></a>
<a class="partner26" style="margin-right:0;"></a> <a class="partner26"></a>
<a class="partner31"></a> <a class="partner31"></a>
<a class="partner32"></a> <a class="partner32"></a>
<a class="partner33"></a> <a class="partner33"></a>
<a class="partner34"></a> <a class="partner34"></a>
<a class="partner35"></a> <a class="partner35"></a>
<a class="partner36" style="margin-right:0;"></a> <a class="partner36"></a>
<a class="partner41"></a> <a class="partner41"></a>
<a class="partner42"></a> <a class="partner42"></a>
<a class="partner43"></a> <a class="partner43"></a>
<a class="partner44"></a> <a class="partner44"></a>
<a class="partner45"></a> <a class="partner45"></a>
<a class="partner46" style="margin-right:0;"></a> <a class="partner46"></a>
<a class="partner51"></a> <a class="partner51"></a>
<a class="partner52"></a> <a class="partner52"></a>
<a class="partner53"></a> <a class="partner53"></a>
<a class="partner54"></a> <a class="partner54"></a>
<a class="partner55"></a> <a class="partner55"></a>
<a class="partner56" style="margin-right:0;"></a> <a class="partner56"></a>
<a class="partner61"></a> <a class="partner61"></a>
<a class="partner62"></a> <a class="partner62"></a>
<a class="partner63"></a> <a class="partner63"></a>
<a class="partner65"></a> <a class="partner65"></a>
<a class="partner66"></a> <a class="partner66"></a>
<a class="partner71" style="margin-right:0;"></a>
<a class="partner72"></a> <a class="partner72"></a>
<a class="partner73"></a> <a class="partner73"></a>
</div> </div>
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.open.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityBasePriceDao extends MongoDao<CommunityBasePrice> {
}
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.open.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.CountDistrictStatBycity;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityPoiDao extends MongoDao<CommunityPoi> {
}
@@ -0,0 +1,10 @@
package com.fangjia.dal.mongo.open.dao.impl;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public class CommunityBasePriceDaoImpl extends MongoDaoImpl<CommunityBasePrice> implements CommunityBasePriceDao {
}
@@ -0,0 +1,10 @@
package com.fangjia.dal.mongo.open.dao.impl;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.open.dao.ApiVisitCountDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.dal.mongo.tool.model.ApiVisitCount;
public class CommunityPoiDaoImpl extends MongoDaoImpl<CommunityPoi> implements CommunityPoiDao {
}
@@ -0,0 +1,74 @@
package com.fangjia.dal.mongo.open.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
/**
* Ð¡Çø½¨ÐÐÖÃÐŶÈÐÅÏ¢
*
* @author shenjian@fangjia.com
*/
@Entity
@Table(name = "business.community_base_price_gj")
public class CommunityBasePrice implements MongoEntity {
private static final long serialVersionUID = -8010010773207622878L;
private String id;
private String city;
private String region;
private String name;
private Double credit;
@Id
@Column(name = "_id")
@Override
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "credit")
public Double getCredit() {
return credit;
}
public void setCredit(Double credit) {
this.credit = credit;
}
}
@@ -0,0 +1,135 @@
package com.fangjia.dal.mongo.open.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
import java.util.Date;
/**
* 小区周边配套统计信息
*
* @author shenjian@fangjia.com
*/
@Entity
@Table(name = "fangjia_stat.community_poi_gj")
public class CommunityPoi implements MongoEntity {
private static final long serialVersionUID = -7663644539542318409L;
private String id;
private String city;
private String region;
private String name;
private Integer subway;
private Integer traffic;
private Integer bank;
private Integer primary;
private Integer junior;
private Integer hospital;
/**
* 不利设施,丧葬
*/
private Integer funeral;
@Id
@Column(name = "_id")
@Override
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "subway")
public Integer getSubway() {
return subway;
}
public void setSubway(Integer subway) {
this.subway = subway;
}
@Column(name = "traffic")
public Integer getTraffic() {
return traffic;
}
public void setTraffic(Integer traffic) {
this.traffic = traffic;
}
@Column(name = "bank")
public Integer getBank() {
return bank;
}
public void setBank(Integer bank) {
this.bank = bank;
}
@Column(name = "primary")
public Integer getPrimary() {
return primary;
}
public void setPrimary(Integer primary) {
this.primary = primary;
}
@Column(name = "junior")
public Integer getJunior() {
return junior;
}
public void setJunior(Integer junior) {
this.junior = junior;
}
@Column(name = "hospital")
public Integer getHospital() {
return hospital;
}
public void setHospital(Integer hospital) {
this.hospital = hospital;
}
@Column(name = "funeral")
public Integer getFuneral() {
return funeral;
}
public void setFuneral(Integer funeral) {
this.funeral = funeral;
}
}
@@ -67,12 +67,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
must.add(addTerms("district_name", alias)); must.add(addTerms("district_name", alias));
must.add(addMatch("cat", "sell")); must.add(addMatch("cat", "sell"));
must.add(addRange("avg_price", minPrice, maxPrice)); must.add(addRange("avg_price", minPrice, maxPrice));
must.add(addRangeGte("p_date", 20210501));
JSONObject term = new JSONObject();
JSONObject match = new JSONObject();
match.put("p_date", 20210501);
term.put("match", match);
must.add(term);
bool.put("must", must); bool.put("must", must);
query.put("bool", bool); query.put("bool", bool);
@@ -168,6 +163,16 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
return term; return term;
} }
private JSONObject addRangeGte(String name, Number min){
JSONObject term = new JSONObject();
JSONObject range = new JSONObject();
JSONObject rangeValue = new JSONObject();
rangeValue.put("gte", min);
range.put(name, rangeValue);
term.put("range", range);
return term;
}
private List<Seawater> wrapSeawater(List<JSONObject> jsonList){ private List<Seawater> wrapSeawater(List<JSONObject> jsonList){
List<Seawater> seawaterList= new ArrayList<Seawater>(); List<Seawater> seawaterList= new ArrayList<Seawater>();
@@ -210,6 +215,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
seawater.setFloor(source.getInt("floor")); seawater.setFloor(source.getInt("floor"));
}catch(Exception e){ }catch(Exception e){
seawater.setFloor(0); seawater.setFloor(0);
}
try{
seawater.setFloorInfo(source.getString("floor_info"));
}catch(Exception e){
} }
if(source.containsKey("fitment")){ if(source.containsKey("fitment")){
seawater.setFitment(source.getString("fitment")); seawater.setFitment(source.getString("fitment"));
@@ -48,6 +48,7 @@ public class Seawater implements SeawaterEntity {
private Double area; private Double area;
private Integer height; private Integer height;
private Integer floor; private Integer floor;
private String floorInfo;
private String fitment; private String fitment;
private Integer room; private Integer room;
private Integer hall; private Integer hall;
@@ -198,6 +199,15 @@ public class Seawater implements SeawaterEntity {
this.floor = floor; this.floor = floor;
} }
@Column(name = "floorInfo")
public String getFloorInfo() {
return floorInfo;
}
public void setFloorInfo(String floorInfo) {
this.floorInfo = floorInfo;
}
@Column(name = "fitment") @Column(name = "fitment")
public String getFitment() { public String getFitment() {
return fitment; return fitment;
@@ -31,12 +31,15 @@ import javax.validation.Valid;
import com.fangjia.core.util.*; import com.fangjia.core.util.*;
import com.fangjia.dal.*; import com.fangjia.dal.*;
import com.fangjia.dal.mongo.model.*; import com.fangjia.dal.mongo.model.*;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.open.dto.*; import com.fangjia.open.dto.*;
import com.fangjia.service.district.*; import com.fangjia.service.district.*;
import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.mongo.fangjia.*;
import com.fangjia.service.open.*; import com.fangjia.service.open.*;
import com.fangjia.service.open.impl.CommunityPoiServiceImpl;
import com.fangjia.service.util.DistrictScoreUtil; import com.fangjia.service.util.DistrictScoreUtil;
import net.sf.json.JSONArray; import net.sf.json.JSONArray;
import net.sf.json.JSONObject; import net.sf.json.JSONObject;
@@ -175,6 +178,9 @@ public class PropertyController {
private SeawaterDao seawaterDao; private SeawaterDao seawaterDao;
private SeawaterService seawaterService; private SeawaterService seawaterService;
private SqmetreRentService sqmetreRentService; private SqmetreRentService sqmetreRentService;
private CommunityPoiService communityPoiService;
private CommunityBasePriceService communityBasePriceService;
private EvaluateFactorService evaluateFactorService;
private FileDao fileDao = FileDao.getInstance(); private FileDao fileDao = FileDao.getInstance();
private static final int DEFAULT_PAGE_SIZE = 20; private static final int DEFAULT_PAGE_SIZE = 20;
@@ -390,6 +396,27 @@ public class PropertyController {
put("5000", "five");// 五线 put("5000", "five");// 五线
} }
}; };
/**
* 第三方均价映射
*/
private static final Map<String, String> OTHER_PRICE_MAP = new HashMap<String, String>() {
{
put("zhuge_price", "诸葛找房");
put("anjuke_price", "安居客");
put("fang_price", "房天下");
put("lianjia_price", "链家");
put("cityhouse_price", "禧泰数据");
put("maitian_price", "麦田");
put("ke_price", "贝壳");
put("taofang_price", "淘房365");
put("zhongyuan_price", "中原");
put("tbs_price", "兔博士");
put("qfang_price", "Q房");
put("fangtan_price", "房探");
put("fangduoduo_price", "房多多");
put("wiwj_price", "我爱我家");
}
};
/** /**
* 利用百度API把GPS转换成Google经纬度 * 利用百度API把GPS转换成Google经纬度
@@ -992,6 +1019,32 @@ public class PropertyController {
} }
} }
// 调整面积不同面积价格
try {
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
int orgAvgPrice = 0;
if (districtStatBo != null) {
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
} else if (districtStatBo.getDistrictStat() != null) {
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
}
}
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
}
} catch (Exception ex) {
}
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0 if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) { && (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
Map<String, Object> districtPriceMap = new HashMap<String, Object>(); Map<String, Object> districtPriceMap = new HashMap<String, Object>();
@@ -1162,19 +1215,52 @@ public class PropertyController {
evaluationService.remove(evaluation.getId()); evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
return mav; return mav;
}
// 调整面积不同面积价格
try {
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
int orgAvgPrice = 0;
if (districtStatBo != null) {
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
} else if (districtStatBo.getDistrictStat() != null) {
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
}
}
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
}
} catch (Exception ex) {
} }
Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>(); Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>();
districtPriceMap.put("city", district.getCity()); districtPriceMap.put("city", district.getCity());
districtPriceMap.put("district", district.getRegion()); districtPriceMap.put("district", district.getRegion());
districtPriceMap.put("name", district.getName()); districtPriceMap.put("name", district.getName());
long totalPrice = evaluation.getBigTotalPrice(); long totalPrice = evaluation.getBigTotalPrice();
if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){ // 取消系数下调
totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity()))); // if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
evaluation.setBigTotalPrice(totalPrice); // totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea())); // evaluation.setBigTotalPrice(totalPrice);
} // evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
// }
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
districtPriceMap.put("unitPrice", evaluation.getAvgPrice()); districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
// 新增置信度
CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId());
if (communityBasePrice != null) {
districtPriceMap.put("credit", communityBasePrice.getCredit());
} else {
districtPriceMap.put("credit", evaluation.getCredit());
}
List<String> names = new ArrayList<String>(); List<String> names = new ArrayList<String>();
names.add(district.getName()); names.add(district.getName());
if(district.getAlias() != null){ if(district.getAlias() != null){
@@ -1186,6 +1272,31 @@ public class PropertyController {
} }
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8); Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2); Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9);
Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1);
// 新增第三方均价
try {
double dCredit = 0.61;
Map<String, Object> rMap = getLastEvaluationPrice(district, evaluation, dCredit);
if (null != rMap) {
if (rMap.containsKey("other_price")) {
Map<String, Integer> otherPriceMap = (Map<String, Integer>) rMap.get("other_price");
Map<String, Integer> priceResultMap = new HashMap<String, Integer>();
for (String key : otherPriceMap.keySet()) {
Integer price = otherPriceMap.get(key);
// 剔除偏差大于10的
if (min10Price <= price && max10Price >= price) {
if (OTHER_PRICE_MAP.containsKey(key)) {
priceResultMap.put(OTHER_PRICE_MAP.get(key), price);
}
}
}
districtPriceMap.put("otherPrice", priceResultMap);
}
}
} catch (Exception ex) {
}
List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20); List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
if(houses.size() > 3){ if(houses.size() > 3){
districtPriceMap.put("calcType", 1); districtPriceMap.put("calcType", 1);
@@ -1214,7 +1325,7 @@ public class PropertyController {
}else if(district.getLng() != null && district.getLng() > 0){ }else if(district.getLng() != null && district.getLng() > 0){
double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6); double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6);
double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6); double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6);
List<District> districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 4, null); List<District> districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 6, null);
if(districts.size() < 2){ if(districts.size() < 2){
evaluationService.remove(evaluation.getId()); evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
@@ -1252,9 +1363,15 @@ public class PropertyController {
districtDto.append("buildYear", String.valueOf(buildYear)); districtDto.append("buildYear", String.valueOf(buildYear));
} }
districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d)); districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d));
if(districtAnalyData != null && districtAnalyData.getHalfYearMarkup() != null){ if(districtAnalyData != null){
districtDto.append("halfYearMarkup",DataUtil.roundDouble( districtAnalyData.getHalfYearMarkup() * 100 ,2)); if (districtAnalyData.getHalfYearMarkup() != null) {
districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2));
}
if (districtAnalyData.getAvgPrice() != null && minPrice <= districtAnalyData.getAvgPrice() && maxPrice >= districtAnalyData.getAvgPrice()) {
districtDto.append("avgPrice", districtAnalyData.getAvgPrice());
}
} }
nearCommunityDtos.add(districtDto); nearCommunityDtos.add(districtDto);
} }
Integer buildYear = districtService.getBuildYear(district); Integer buildYear = districtService.getBuildYear(district);
@@ -1262,6 +1379,18 @@ public class PropertyController {
modelFeaturesFto.put("buildYear", String.valueOf(buildYear)); modelFeaturesFto.put("buildYear", String.valueOf(buildYear));
} }
modelFeaturesFto.append("nearCommunity", nearCommunityDtos); modelFeaturesFto.append("nearCommunity", nearCommunityDtos);
// 模型特征新增周边数量
CommunityPoi communityPoi = communityPoiService.getById(district.getId());
if (communityPoi != null) {
modelFeaturesFto.put("subway", communityPoi.getSubway());
modelFeaturesFto.put("traffic", communityPoi.getTraffic());
modelFeaturesFto.put("bank", communityPoi.getBank());
modelFeaturesFto.put("primary", communityPoi.getPrimary());
modelFeaturesFto.put("hospital", communityPoi.getHospital());
modelFeaturesFto.put("junior", communityPoi.getJunior());
modelFeaturesFto.put("funeral", communityPoi.getFuneral());
}
districtPriceMap.put("modelFeatures", modelFeaturesFto); districtPriceMap.put("modelFeatures", modelFeaturesFto);
} }
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap)); mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
@@ -5627,4 +5756,16 @@ public class PropertyController {
public void setRentDealPriceService(RentDealPriceService rentDealPriceService) { public void setRentDealPriceService(RentDealPriceService rentDealPriceService) {
this.rentDealPriceService = rentDealPriceService; this.rentDealPriceService = rentDealPriceService;
} }
public void setCommunityPoiService(CommunityPoiService communityPoiService) {
this.communityPoiService = communityPoiService;
}
public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) {
this.evaluateFactorService = evaluateFactorService;
}
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
this.communityBasePriceService = communityBasePriceService;
}
} }
@@ -75,8 +75,8 @@ public class AnalyticsFilter implements Filter {
if ("1".equals(request.getParameter("__refresh_cache"))) { if ("1".equals(request.getParameter("__refresh_cache"))) {
threadLocalData.put(ThreadLocalKey.DisableCache, "1"); threadLocalData.put(ThreadLocalKey.DisableCache, "1");
} }
createUID(request, response); //createUID(request, response);
createSID(request, response); //createSID(request, response);
response.addHeader("Access-Control-Allow-Origin", "*"); response.addHeader("Access-Control-Allow-Origin", "*");
response.addHeader("Access-Control-Allow-Methods", "*"); response.addHeader("Access-Control-Allow-Methods", "*");
chain.doFilter(request, response); chain.doFilter(request, response);
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{ class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) { static main(args) {
runBatch(new BigdataCommunityBatch(), args); runBatch(new BigdataCommunityBatch(), args);
@@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{
}else{ }else{
def jsonObj = null def jsonObj = null
if(dist.address){ if(dist.address){
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name) // jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
} }
if(jsonObj && jsonObj.searchAddress){ if(jsonObj && jsonObj.street){
ret << jsonObj.searchAddress.street if(jsonObj.street){
ret << jsonObj.searchAddress.streetNumber ret << jsonObj.street
}else{
ret << ""
}
if(jsonObj.streetNumber){
ret << jsonObj.streetNumber
}else{
ret << ""
}
}else{ }else{
ret << "" ret << ""
ret << "" ret << ""
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city)) // def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220318, 20220419); exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果 //测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
totalCount.addAndGet(1) totalCount.addAndGet(1)
def commId = rMap.get("_id").toString() def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias) def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要 // if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
@@ -408,7 +408,7 @@ class BigdataForsaleBatch extends BaseBatch{
} }
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false); QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -449,6 +449,13 @@ class BigdataForsaleBatch extends BaseBatch{
def map = houses[k]; def map = houses[k];
DBObject seawater = new BasicDBObject(map) DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="ÆÕͨסլ"){
if(seawater.estate_type&& seawater.estate_type.contains("ÉÌס")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
}
if(!seawater.total_price){ if(!seawater.total_price){
continue; continue;
} }
@@ -57,6 +57,12 @@ class CcbFtpUtils {
ftpUpload(dataCodeFileName)//上传加密码文件到FTP ftpUpload(dataCodeFileName)//上传加密码文件到FTP
} }
static def listFileNames(month){
//上传到ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
return files
}
static main(args) { static main(args) {
@@ -15,8 +15,8 @@ class DealPriceDataProcess {
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", // def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") // "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -373,16 +373,17 @@ class DealPriceDataProcess {
def citys = CcbCommonUtils.citys def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01' def publishTime = '2021-10-01'
def month = 202203 def month = 202204
def rows = [] def rows = []
for(def city : citys) { for(def city : citys) {
def rets = findPeGteThirdParty(city, 202204) def rets = findPeGteThirdParty(city, 202205)
def total = 0 def total = 0
println city +" "+rets.size()
for(def m : rets){ for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}" def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance236.executeUpdate(sql) sqlInstance236.executeUpdate(sql)
} }
println city +" "+rets.size()
} }
} }
@@ -38,8 +38,8 @@ class ExportOracleData {
exportCommunityBuileType() exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth) exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-03-19") exportBigdataForsale("2022-04-19")
exportBigdataDeal("2022-03-01") exportBigdataDeal("2022-04-01")
//计算结果相关 //计算结果相关
@@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql import groovy.sql.Sql
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
import org.apache.commons.codec.digest.DigestUtils
class ExportOracleEvaluationData { class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
@@ -33,6 +35,8 @@ class ExportOracleEvaluationData {
exportCommCaseDetailGj(curMonth) exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28" def curDate = curMonth.toString()+"28"
// 生成MD5文件
exportMd5()
SSHHelper jschCommand = new SSHHelper(); SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// okÎļþ // okÎļþ
@@ -413,4 +417,23 @@ class ExportOracleEvaluationData {
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate) CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
} }
def exportMd5(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
Vector files = CcbFtpUtils.listFileNames(curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
}
}
def fileName = "/ccb/md5.txt"
if(CommonDatautils.isWindow()){
fileName = "D:\\房价网\\md5.txt"
}
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "md5.txt 上传Ftp完成......"
}
} }
@@ -1,21 +1,29 @@
package com.fangjia.script.ccbevaluation package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import groovy.sql.Sql
import org.bson.types.ObjectId
class LoadGuide { class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) { static main(args) {
def task = new LoadGuide() def task = new LoadGuide()
task.run() task.run()
} }
def run(){ def run(){
addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx") // saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
@@ -26,9 +34,51 @@ class LoadGuide {
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
} }
def addPersonGuided(filename){
ExcelUtils.modify(filename, { d ->
def city = d['B']
if(city){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
createSql( d, cityCodeDatas, distCodeMap)
}
})
}
def createSql(d, cityCodeDatas, distCodeMap){
def id = new ObjectId()
def commid = d['A']
def cityId = cityCodeDatas.get(d['B'])
def sampleRegion = d['C'].replace("街道","").replace("镇","")
def distRegion = distCodeMap.get(sampleRegion)
def cityName = d['B']
def distName = d['C']
def commName = d['D']
def price = d['E'].toDouble().toInteger()
def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district")
if(dist){
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
def dealDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " +
"('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')"
println sql
sqlInstance.executeUpdate(sql)
}else{
println " 没查到 "+commName
}
}
def match(def filename){ def match(def filename){
def city = "西安" def city = "西安"
ExcelUtils.modify(filename,{d-> ExcelUtils.modify(filename,{d->
@@ -0,0 +1,67 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DistrictCodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* oracle中小区统计数据同步至85中
*/
class OracleDistrict2Mongo85 {
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017)
static main(args) {
// 只写入重庆市数据
def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ"
def count = 0
sqlInstance.eachRow(sql, { r ->
def city = DistrictCodeUtils.getCityName(r['cityid'])
def region = RegionData.formatRegion(city, r['distname'])
DBObject dbObject = new BasicDBObject("_id", r['commid'])
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
.append("name", r['commname'])
.append('subway', Integer.parseInt(r['subway'].toString()))
.append("traffic", Integer.parseInt(r['traffic'].toString()))
.append("bank", Integer.parseInt(r['bank'].toString()))
.append("primary", Integer.parseInt(r['primary'].toString()))
.append("junior", Integer.parseInt(r['junior'].toString()))
.append("hospital", Integer.parseInt(r['hospital'].toString()))
.append("funeral", Integer.parseInt(r['funeral'].toString()))
.append("c_date", new Date());
communityPoiConn.update(dbObject, updateObject, true, false)
count++;
if (count % 500 == 0) {
println "已写入" + count + "条小区数据"
}
});
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DistrictCodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
/**
* oracle中小区置信度数据同步至85中
*/
class OracleDistrictBasePrice2Mongo85 {
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017)
static main(args) {
def sql = "SELECT cityname, distname, commid, commname, confidence FROM community_base_price_gj WHERE YEAR_MONTH='202205'"
def count = 0
sqlInstance.eachRow(sql, { r ->
def city = r['cityname']
def region = RegionData.formatRegion(city, r['distname'])
DBObject dbObject = new BasicDBObject("_id", r['commid'])
if (r['confidence'] != null) {
def credit = new BigDecimal(r['confidence']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble()
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
.append("name", r['commname'])
.append("credit", credit)
.append("c_date", new Date());
communityBasePriceConn.update(dbObject, updateObject, true, false)
count++;
} else {
print "置信度为空:" + r.toString()
}
if (count % 500 == 0) {
println "已写入" + count + "条小区数据"
}
});
}
}
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch { class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202203//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202201 def startDate = 202202
def endDate = 202203 def endDate = 202204
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) { static main(args) {
def task = new TradeStatLog() def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量 task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交 // task.exportTrade()//导出未入库成交
// task.thirdPartyStat() // task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR) // println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat() // task.averageStat()
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){ def run(){
def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
def rets = [] def rets = []
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01" //def startDate = "2021-05-01"
def startDate = "2022-01-01" def startDate = "2022-02-01"
def date = "2022-02-01" def date = "2022-03-01"
def endDate = "2022-03-01" def endDate = "2022-04-01"
def newDate = "2022-04-01" def newDate = "2022-05-01"
for(def city : citys){ for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
} }
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-03成交统计.xlsx", rets, "数据") ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
println "表格生成完成" println "表格生成完成"
} }
@@ -0,0 +1,155 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class CcbDataStat {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new CcbDataStat()
// task.run()
task.tradeRun()
}
def tradeRun(){
def ccbCitys = CcbCommonUtils.citys;
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d ->
if(d['A']){
if(ccbCitys.contains(d['A'])){
d['D'] = findOracleTotalCount(d['A'])
}else{
def totalcount = findAllTradeCount(d['A'], startDate, endDate)
def repeatCount = findRepeatCount(d['A'], startDate, endDate)
def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate)
def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate)
d['B'] = totalcount
d['C'] = repeatCount
d['D'] = matchTotalCount
d['E'] = matchRepeatTotalCount
}
}
})
}
def findRepeatCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def set = new HashSet()
for(def d : dbcursor){
def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
if(!set.contains(key)){
set << key
}
}
return set.size()
}
def findMatchRepeatTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def dbcursor = DEAL_PRICE_COLL.find(q)
def set = new HashSet()
for(def d : dbcursor){
def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
if(!set.contains(key)){
set << key
}
}
return set.size()
}
def findMatchTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
public static def findAllTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
// q.put("match_id", new BasicDBObject('$exists', false))
def total = DEAL_PRICE_COLL.count(q)
return total
}
def findOracleTotalCount(city){
def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate<to_date('2022-01-01','yyyy-MM-dd')";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def run(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
if(d['A']) {
def citycode = cityCodeDataMap.get(d['A'])
def total = findTotalCount(citycode)
def f_count = findDistFinishDate(citycode)
def b_count = findBuildTypeCount(citycode)
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
d['C'] = f_pre
d['D'] = b_pre
d['E'] = total
println d['A'] +" "+ total +" "+f_count+" "+b_count
}
})
}
def findTotalCount(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findDistFinishDate(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findBuildTypeCount(citycode){
def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
}
@@ -0,0 +1,117 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
class CcbDistNameMatch {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new CcbDistNameMatch();
// task.run()
task.run1()
}
def run1(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
if(d['D'].contains("、")){
println d['D']
}
def arrs = d['D'].split("、")
def list = []
def flags = []
for(def s : arrs){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district")
if(dist){
list << dist.name
flags << "完全一致"
}else{
def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion)
if(jsonObject != null && jsonObject.precise!=-1) {
if(jsonObject.get("district") == newRegion){
list << jsonObject.get("name")
flags << "匹配接口"
}
}
}
}
if(list){
list = list.unique()
d['E'] = list.join(",")
d['F'] = flags.join(",")
}
}
}
})
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d ->
def regionCode = null
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
d['D'] = dist.name
def count = findDistrictByOracle(dist._id.toString())
if(count && count>0){
d['F'] = 1
}else{
d['H'] = dist.estate_type2
d['J'] = dist.name_cat
}
}else{
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
if (rets) {
if(rets.size()>1){
d['E'] = "别名-多个"
}else{
dist = rets[0]
d['E'] = d['C'] +"--"+dist.name
def count = findAliasByOracle(dist._id.toString())
if(count && count>0) {
d['G'] = 1
}else{
d['H'] = dist.estate_type2
d['J'] = dist.name_cat
}
}
}
}
}
}
});
}
def findDistrictByOracle(commid){
def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findAliasByOracle(commid){
def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
}
@@ -7,6 +7,7 @@ import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
@@ -26,9 +27,9 @@ class CcbDistrictToDB {
// task.run() // task.run()
// task.run1() // task.run1()
// task.meituanRun() // task.meituanRun()
// task.meituanSqrentRun() task.meituanSqrentRun()
// task.updateDistrict() // task.updateDistrict()
task.updateDongguangRegion() // task.updateDongguangRegion()
} }
def updateDongguangRegion(){ def updateDongguangRegion(){
@@ -80,13 +81,17 @@ class CcbDistrictToDB {
} }
def meituanSqrentRun(){ def meituanSqrentRun(){
ExcelUtils.modify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> def index = 0
ExcelUtils.modify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
index++
println index+" "+d['H']
def city = d['F'] def city = d['F']
def region = d['G'] def region = d['G']
def newRegion = RegionData.formatRegion(city, region) def newRegion = RegionData.formatRegion(city, region)
if(newRegion) { if(newRegion) {
def name = d['H'] def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name) name = nameFileter(name)
if(name){ if(name){
def estateType2 = "普通住宅" def estateType2 = "普通住宅"
if(name.contains("别墅")){ if(name.contains("别墅")){
@@ -103,7 +108,6 @@ class CcbDistrictToDB {
} }
} }
} }
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0) CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
} }
@@ -115,97 +119,100 @@ class CcbDistrictToDB {
def index = 0//美团需入库小区合并后.xlsx def index = 0//美团需入库小区合并后.xlsx
def keys = new CopyOnWriteArrayList() def keys = new CopyOnWriteArrayList()
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团需入库小区合并后.xlsx", { d -> ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
index++
println index+" "+d['H']
def city = d['F'] def city = d['F']
def region = d['G'] def region = d['G']
def newRegion = RegionData.formatRegion(city, region) def newRegion = RegionData.formatRegion(city, region)
if(newRegion){ if(newRegion) {
def name = d['H'] def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name) name = nameFileter(name)
if (name) {
def formatAddress = null def formatAddress = null
if(d['E']){ if (d['I']) {
formatAddress = AddressUtil.formatAddress(city, d['E']) formatAddress = AddressUtil.formatAddress(city, d['I'])
} }
if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){ // if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) // def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
def map = findStandardObject2(city,region, name, d['E'], rets); // def map = findStandardObject2(city,region, name, d['E'], rets);
if(map && map.address){ // if(map && map.address){
formatAddress = AddressUtil.formatAddress(city, map.address) // formatAddress = AddressUtil.formatAddress(city, map.address)
if(!formatAddress){ // if(!formatAddress){
if(map.address.contains("交叉口")){ // if(map.address.contains("交叉口")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3) // formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
}else if(map.address.contains("交叉")){ // }else if(map.address.contains("交叉")){
formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2) // formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
} // }
} // }
} // }
} // }
Long glat = null Long glat = null
Long glng = null Long glng = null
Long blat = null Long blat = null
Long blng = null Long blng = null
if(d['K']){ if (d['L']) {
glat = d['L'].toString().toDouble() * 10000000000L glat = d['M'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L glng = d['L'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3) def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L blat = baidu.y * 10000000000L
} }
def estateType2 = "普通住宅" def estateType2 = "普通住宅"
if(name.contains("别墅")){ if (name.contains("别墅")) {
estateType2 = "别墅" estateType2 = "别墅"
}else if(name.contains("商住")){ } else if (name.contains("商住")) {
estateType2 = "商住" estateType2 = "商住"
}else{ } else {
def type = d['I'] def type = d['J']
if(type){ if (type) {
if(type.contains("别墅")){ if (type.contains("别墅")) {
estateType2 = "别墅" estateType2 = "别墅"
}else if(type.contains("商住")){ } else if (type.contains("商住")) {
estateType2 = "商住" estateType2 = "商住"
} }
} }
} }
def key = city+"##"+newRegion+"##"+name def key = city + "##" + newRegion + "##" + name
if(name && !keys.contains(key)){ if (name && !keys.contains(key)) {
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district") def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){ if (!dist) {
keys << key keys << key
def data = new BasicDBObject(); def data = new BasicDBObject();
data.put("name_cat",0) data.put("name_cat", 12)
data.put("status",0) data.put("status", 0)
data.put("cat","district") data.put("cat", "district")
data.put("c_date", new Date()) data.put("c_date", new Date())
data.put("m_date", new Date()) data.put("m_date", new Date())
data.put("city", city) data.put("city", city)
data.put("region",newRegion) data.put("region", newRegion)
data.put("name", name) data.put("name", name)
data.put("estate_type2",estateType2) data.put("estate_type2", estateType2)
data.put("address", formatAddress) data.put("address", formatAddress)
if(glat){ if (glat) {
data.put("lat",glat) data.put("lat", glat)
data.put("lng",glng) data.put("lng", glng)
data.put("lat2",blat) data.put("lat2", blat)
data.put("lng2",blng) data.put("lng2", blng)
} }
def markMap = [:] def markMap = [:]
markMap.put("data_source", "机器入库") markMap.put("data_source", "机器入库")
data.put("mark_map",markMap) data.put("mark_map", markMap)
index++ index++
DbUtils.seaweed.save(data); DbUtils.seaweed.save(data);
println index+" "+data println index + " " + data
def stop = false def stop = false
while (!stop){ while (!stop) {
if(data._id.toString()){ if (data._id.toString()) {
addPrice(data) addPrice(data)
stop = true; stop = true;
} }
@@ -213,19 +220,17 @@ class CcbDistrictToDB {
} }
} }
}
}else{ }else{
println city +" "+region +" 区域无法标准化" println city +" "+region +" 区域无法标准化"
} }
},0); },0);
} }
def addPrice(dist){ def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01") def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25") def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate) def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){ for(def weekend : weekends){
@@ -435,9 +440,12 @@ class CcbDistrictToDB {
} }
def nameFileter(def name){ def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","") if(name){
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
}
return name; return name;
} }
@@ -1,10 +1,12 @@
package com.fangjia.script.ccbevaluation.data package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.poi.ss.util.RegionUtil import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId import org.bson.types.ObjectId
import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.BaiDuApiUtils
@@ -13,7 +15,82 @@ class DistrictMarge {
static main(args) { static main(args) {
def task = new DistrictMarge() def task = new DistrictMarge()
//task.run() //task.run()
task.repeatAddress() // task.repeatAddress()
// task.removeDist()
task.updateAddress()
// task.addMappingAlias()
}
def updateAddress(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d ->
if (d['A']) {
def region = d['B'].trim()
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
DBObject udata = new BasicDBObject()
udata.put("address", d['D'])
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
});
}
def removeDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
DBObject udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
})
}
def addMappingAlias(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
index ++
if (d['A']) {
def region = d['B'].trim()
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
def mapping_alias = dist.mapping_alias
if(!dist.mapping_alias){
mapping_alias = []
}
def alias = d['D']
def arrs = alias.split("、")
for(def s : arrs){
if(!mapping_alias.contains(s)){
mapping_alias << s
}
}
DBObject udata = new BasicDBObject()
udata.put("mapping_alias", mapping_alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
})
} }
def repeatAddress(){ def repeatAddress(){
@@ -36,10 +36,37 @@ class MatchMapData {
// task.run3() // task.run3()
// task.exportHouse() // task.exportHouse()
// task.addDistrictLevel() // task.addDistrictLevel()
task.run4() // task.run4()
task.run5()
} }
def run5(){
ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\Êý¾ÝºËʵ.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
if(dist.mapping_alias){
d['D'] = dist.mapping_alias.join(",")
}
d['E'] = dist.address
}
def dist1 = CommonDatautils.findDistrict(d['A'], newRegion, d['G'], "district")
if(dist1){
if(dist1.mapping_alias){
d['H'] = dist1.mapping_alias.join(",")
}
d['I'] = dist1.address
}
}
}
});
}
def run4() { def run4() {
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d -> ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
def regionCode = null def regionCode = null
@@ -0,0 +1,108 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class OracleHouse {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new OracleHouse()
task.run()
}
def run(){
def rets = findAllHouse("2022-04-19")
def totalCount = new AtomicLong()
println rets.size()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
def arrs = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def flag = findHouseFlag(arrs[0])
println totalCount.get()+" "+ arrs[0] +" "+flag
if(flag){
def dist = CommonDatautils.findDistrictById(arrs[1])
if(dist && dist.estate_type2=="ÆÕͨסլ"){
removeHouse(arrs[0])
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findHouseFlag(def id){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_comp_potential", queryBuilder);
def delFlag = false
for(def h : houses){
if(h.estate_type && (h.estate_type.contains("ÉÌ") || h.estate_type.contains("°ì"))){
println id+" "+h.estate_type
delFlag = true
}
}
return delFlag
}
def findAllHouse(def publishDate) {
def sql = "select FORSALEID,COMMID from bigdata_forsale_gj where publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
def allRows = []
def index = 0
sqlInstance.eachRow(sql, { r ->
index = index + 1
println index
def data = []
data << r.FORSALEID
data << r.COMMID
allRows << data
});
return allRows
}
def removeHouse(houseid){
def sql = "delete from bigdata_forsale_gj where FORSALEID='${houseid}' ";
sqlInstance.executeUpdate(sql)
}
}
@@ -2,39 +2,70 @@ package com.fangjia.script.chongqing
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.paimai.DataUtil import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.OpenApiClient import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql import groovy.sql.Sql
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CopyOnWriteArrayList
import java.util.zip.GZIPInputStream
class MatchEvaluateRegionTest { class MatchEvaluateRegionTest {
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\美团\\10城小区明细-V10.xlsx"
def total = 1500
static main(args) { static main(args) {
def task = new MatchEvaluateRegionTest() def task = new MatchEvaluateRegionTest()
task.run()//小区匹配 // task.run()//小区匹配
task.run1()//地图匹配 // task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 // task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice() // task.addPrice()
task.addAuthor() // task.addArroundPrice()
// task.addArroundPriceByLat()
// task.addRentRate()
// task.addAuthor()
// task.addLevel()
// task.addSorce()
// task.addArroundScore()
task.pkPrice()
} }
def run(){ def run(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
def index = 0
ExcelUtils.multiThreadingModify(path, { row -> ExcelUtils.multiThreadingModify(path, { row ->
index ++
if(!row['F']){ if(!row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], "property", false, null, null) println index+" "+row['E']
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false,row['B'],null)
if(jsonObject != null && jsonObject.precise!=-1) { if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id") row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city") row['G'] = jsonObject.get("city")
@@ -43,18 +74,17 @@ class MatchEvaluateRegionTest {
row['J'] = jsonObject.get("address") row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType") row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map") row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng") row['M'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat") row['N'] = jsonObject.get("lat")
} }
} }
}); });
} }
def run1(){ def run1(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row -> ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){ if(!row['F'] && row['A']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['D'], null, true, null, null) def jsonObject = OpenApiClient.addressMatch(row['A'], row['E'], null, true, row['B'],row['C'].toString())
if(jsonObject != null && jsonObject.precise!=-1) { if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id") row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city") row['G'] = jsonObject.get("city")
@@ -63,52 +93,104 @@ class MatchEvaluateRegionTest {
row['J'] = jsonObject.get("address") row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType") row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map") row['L'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng") row['M'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat") row['N'] = jsonObject.get("lat")
} }
} }
}); });
} }
def pkPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['R'] && row['R'].toString().toDouble().toInteger()==1){
def obj = CommonDatautils.queryDistrictPrice(row['G'], row['H'], row['I'], DateUtils.parseFromTightByPattern("2022-05-22","yyyy-MM-dd"))
if(!obj){
row['R'] = ''
row['S'] = "无最新价"
}
}
});
}
def addPrice(){ def addPrice(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx"
ExcelUtils.multiThreadingModify(path, { row -> ExcelUtils.multiThreadingModify(path, { row ->
if(!row['L']){ if(!row['L']){
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I']) def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
if(priceObj){ if(priceObj){
row['M'] = priceObj.avg_price row['O'] = priceObj.avg_price
row['N']= (int)(priceObj.avg_price * row['E']) // row['N']= (int)(priceObj.avg_price * row['E'])
} }
def avg_price = findAvgPrice(row['M']) def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['G'], row['H'], row['I'])
if(row['M'] && avg_price){ if(sqmetreRentPrice){
def weekendPrice = row['M'].toString().toDouble() row['P'] = sqmetreRentPrice
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
} }
// def avg_price = findAvgPrice(row['F'])
// if(row['M'] && avg_price){
// def weekendPrice = row['M'].toString().toDouble()
// def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
// row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
// }
} }
}); });
} }
def addRentRate(){
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
def radioMap = [:]
for(def city : citys) {
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
radioMap.put(city, avg_radio)
}
ExcelUtils.multiThreadingModify(path, { row ->
def city = row['G']
if (row['F'] && row['O'] && !row['P']) {
def avg_rent_radio = radioMap.get(city)
if(avg_rent_radio){
row['P'] = (int)(row['O']/avg_rent_radio)
row['S'] = "租金推算1"
}
}
})
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['O']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['E'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['O'] = priceObj.unitPrice
// row['N'] = (int) (priceObj.unitPrice * row['E'])
row['S'] = "周边价格"
}
}
});
}
def addAuthor(){ def addAuthor(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\测试模板.xlsx" def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def total = 747
def personTotal = authors.size() def personTotal = authors.size()
def avg = Math.ceil(total/personTotal) def avg = Math.ceil(total/personTotal)
def index = 0 def index = 0
def count = 0; def count = 0;
ExcelUtils.modify(path, { row -> ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){ if(index < avg){
row['P'] = authors[count] row['N'] = authors[count]
} }
index++ index++
if(index >= avg){ if(index >= avg){
count++ count++
index = 0 index = 0
} }
}
}); });
} }
@@ -236,4 +318,295 @@ class MatchEvaluateRegionTest {
}); });
return avg_price return avg_price
} }
def addSorce(){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['F']){
def score = findDistrictScore(d['F'])
if(score){
d['R'] = score
}
}
});
}
def addArroundScore(){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['R'] && d['F']){
def dist = null
if(d['F'].toString().size()>16){
dist = CommonDatautils.findDistrictById(d['F'])
}
if(!dist){
dist = new BasicDBObject()
dist.city = d['G']
dist.lng = d['M'].toString().toDouble()*10000000000L
dist.lat = d['N'].toString().toDouble()*10000000000L
}
if(dist.lat){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
if (list) {
list = sortByDistance(dist, list);
if(list && list.size()>10){
list = list[0..9]
}
if(list){
def scores = []
for(def s : list){
def score = findDistrictScore(s._id.toString())
if(score){
scores << score
}
}
def arround_score =(int)(scores.sum()/scores.size())
d['R'] = arround_score
d['T'] = "周边小区评分2"
}
}
}
}
});
}
def addArroundPriceByLat(){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['O'] && d['F'] && (d['K'].contains("住宅") || d['K'].contains("商住") ||d['K'].contains("别墅"))){
def dist = null
if(d['F'].toString().size()>16){
dist = CommonDatautils.findDistrictById(d['F'])
}
if(!dist){
dist = new BasicDBObject()
dist.city = d['G']
dist.lng = d['M'].toString().toDouble()*10000000000L
dist.lat = d['N'].toString().toDouble()*10000000000L
}
if(dist.lat){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
if (list) {
list = sortByDistance(dist, list);
if(list && list.size()>10){
list = list[0..9]
}
if(list){
def prices = []
for(def s : list){
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def arround_price =(int)(prices.sum()/prices.size())
d['O'] = arround_price
d['S'] = "周边均价3"
}
}
}
}
}
});
}
def sortByDistance(d, list){
def newList = []
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
newList << nb
}
//排序
Collections.sort(newList, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return newList;
}
def findDistrictScore(id){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
def indexObj = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
def score = 0
for(def h : indexObj){
score = h.total_price_rate
if(score){
break;
}
}
return score
}
def addLevel() {
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
ExcelUtils.multiThreadingModify(path, { row ->
def address = row['E']
def city = RegionData.formatCity(row['A'])
if(row['K'] && !row['K'].contains("住宅") && !row['K'].contains("商住") && !row['K'].contains("别墅")){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
}
}
}
})
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
return datas;
}
def getArroundDistrictPriceByAddress(def city, def address, def levels){
def json = GaoDeApiUtils.geo(city, address)
def location = null
def detail = new BasicDBObject()
def level = null
def rank = null
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
location = m.location
level = m.level
}
}
if(location){
def arrs = location.split(",")
detail.lng = arrs[0].toString().toDouble()*10000000000L
detail.lat = arrs[1].toString().toDouble()*10000000000L
}
if(detail.lng){
detail.city = city
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
if(list && list.size()>10){
list = sortByDistance(detail, list);
try{
list = list[0..9]
}catch(Exception e){
println list.size()
}
}
def arroundAvgPrice = getArroundAvgPrice(list)
if(arroundAvgPrice){
def mod = Integer.MAX_VALUE;
for(def d : levels){
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
if(newMod<mod){
mod = newMod
rank = d.level
}
}
}
}
def m = [:]
m.level = level
m.rank = rank
return m
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
return null
}
} }
@@ -28,14 +28,14 @@ class MatchEvaluateTest {
static main(args) { static main(args) {
def task = new MatchEvaluateTest() def task = new MatchEvaluateTest()
// task.run()//小区匹配 task.run()//小区匹配
// task.run1()//地图匹配 task.run1()//地图匹配
//// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除 // task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
// task.addPrice() task.addPrice()
// task.addArroundPrice() task.addArroundPrice()
// task.addAuthor() task.addAuthor()
task.stat() // task.stat()
} }
def addArroundPrice(){ def addArroundPrice(){
@@ -57,7 +57,7 @@ class MatchEvaluateTest {
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row -> ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){ if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null) def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false)
if(jsonObject != null && jsonObject.precise!=-1) { if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id") row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city") row['E'] = jsonObject.get("city")
@@ -77,7 +77,7 @@ class MatchEvaluateTest {
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row -> ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){ if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true, null, null) def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true)
if(jsonObject != null && jsonObject.precise!=-1) { if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id") row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city") row['E'] = jsonObject.get("city")
@@ -117,7 +117,7 @@ class MatchEvaluateTest {
def addAuthor(){ def addAuthor(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx" // def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
def authors = ["王蕾","丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"] def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size() def personTotal = authors.size()
def avg = Math.ceil(total/personTotal) def avg = Math.ceil(total/personTotal)
@@ -0,0 +1,666 @@
package com.fangjia.script.chongqing
import com.fangjia.core.util.SourceNameUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
class PocTest {
def total = 3259
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new PocTest();
task.run();
}
/**
* A 序号
* B 城市
* C 区域
* D 地址
* E 面积
* @return
*/
def run(){
def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
// match(inputfile)
// addPrice(inputfile)
// addArroundPrice(inputfile)
// standard(inputfile)
// addLoudongNum(inputfile)
// addAuthor(inputfile)
// pkPrice(inputfile)
// addPpeValue(inputfile)
// matchMap(inputfile)
// addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
//
// createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
// createPaimai(inputfile,"D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
// createHouse(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
// createDealPrice(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
// createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
}
def addPpeValue(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['F'] && row['R']){
def avg_price = findAvgPrice(row['R'])
if(avg_price){
def ppe15 = Math.abs((row['F'] - avg_price) * 100 / avg_price)
row['Z'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def pkPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def jsonObject = OpenApiClient.webankEvaluate(dist.city, dist.region, dist.name, row['E'].toString())
if(jsonObject != null && jsonObject.code==200) {
def result = jsonObject.result
if(result){
row['Y'] = result.get("unitPrice")
}
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Q'] = jsonObject.get("name")
row['R'] = jsonObject.get("id")
}
}
})
}
def matchMap(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
def propertyType = jsonObject.get("propertyType")
if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){
row['Q'] = jsonObject.get("name")
row['R'] = new ObjectId().toString()
row['W'] = propertyType
def arroundObj = FangjiaClientUtils.arroundPrice(city, address)
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['F'] = priceObj.unitPrice
}
}
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['F'] = priceObj.avg_price
}
}
}
})
}
def addArroundPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['F'] = priceObj.unitPrice
row['W'] = "周边均价"
}
}
});
}
def standard(path){
ExcelUtils.multiThreadingModify(path, { row ->
def city = row['B']
def address = row['D']
if(address){
println "[standard] address:"+address
def jsonObject = OpenApiClient.standard2Action(city, address)
if(jsonObject){
println jsonObject
row['G'] = jsonObject.get("province")
row['H'] = jsonObject.get("city")
row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("township")
row['K'] = jsonObject.get("street")
row['L'] = jsonObject.get("streetNumber")
row['M'] = jsonObject.get("buildingNumber")
row['N'] = jsonObject.get("unitNumber")
row['O'] = jsonObject.get("floor")
row['P'] = jsonObject.get("roomNumber")
}
}
})
}
def addLoudongNum(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
def houseNum = row['M']
def unitNum = row['N']
println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum
if (commid && houseNum) {
def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum)
if (jsonObject && jsonObject.result) {
if(jsonObject.result.size()>=1){
def obj = jsonObject.result[0]
row['T'] = obj.id
row['S'] = obj.houseNumber
}
}
if(row['P']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
row['V'] = obj.id
row['U'] = obj.roomNum
}
}
}
}
})
}
def createLoudong(path, output){
def rets = new CopyOnWriteArrayList()
def datas = ExcelUtils.read(path, 0, 1)
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
for(def row : datas){
def data = []
data << row['B']
data << row['C']
data << row['D']
data << row['E']
if (row['R'] && row['M']) {
def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N'])
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
def obj = jsonObject.result[0]
data << obj.houseNumber
data << obj.id
data << obj.type// 物业类型
data << ""//主朝向
data << obj.floors//地上层数
data << ""//地下层数
data << obj.buildYear;//竣工日期
data << ""//梯
data << ""//户
data << obj.households//总户数
data << ""//景观
}else{
data << ""
data << ""
data << ""// 物业类型
data << ""//主朝向
data << ""//地上层数
data << ""//地下层数
data << ""//竣工日期
data << ""//梯
data << ""//户
data << ""//总户数
data << ""//景观
}
if(row['P']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
data << obj.roomNum//户号
data << obj.id//户ID
data << obj.direction//主
data << obj.sight//景观
if(obj.roomNum && obj.roomNum.contains("跃")){
data << "跃式"//户型结构
}else{
data << ""//户型结构
}
}
}
}
}
rets << data
}
ExcelUtils.write(output, rets, "底层数据-楼栋-户")
}
def addDetail(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
for(def d : datas){
def commid = d['R']
println "[addDetail] commid:"+commid
if(commid){
def dist = CommonDatautils.findDistrictById(commid)
if(dist){
def row = []
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
if(jsonObject) {
row << d['B']
row << d['C']
row << d['D']
row << jsonObject.get("name")//小区名称
row << commid//小区ID
if(jsonObject.get("alias")){
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
}else{
row << ""
}
row << jsonObject.get("address")//小区地址
row << jsonObject.get("commercialArea")//板块
row << jsonObject.get("township")//街道***********
row << jsonObject.get("avgPrice")//基准价
row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度
row << jsonObject.get("finishBuildTime")//竣工时间
row << jsonObject.get("developer")//开发商名称
row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
row << jsonObject.get("propertyManagement")//物业公司
row << jsonObject.get("managementFee")//物业费
row << jsonObject.get("buildingCount")//住宅幢数
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
}
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(result.keySchool){
row << '是'
}else {
row << ""
}
if(result.school){
row << result.school.join(",").replace('"',"")
}else {
row << ""
}
def bus = []
if(result.bus){
bus.addAll(result.bus)
}
if(result.metro){
bus.addAll(result.metro)
}
if(bus){
row << bus.join(",").replace('"',"")
}else{
row << ""
}
if(result.supermarket){
row << result.supermarket.join(",").replace('"',"")
}else{
row << ""
}
if(result.hospital){
row << result.hospital.join(",").replace('"',"")
}else{
row << ""
}
if(result.bank){
row << result.bank.join(",").replace('"',"")
}else{
row << ""
}
}
rows << row
}
}
}
ExcelUtils.write(output, rows, "底层数据-小区")
}
def createPaimai(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
def keys = []
for(def d : datas){
def commid = d['R']
println "[createPaimai] commid:"+commid
if(commid) {
def dist = CommonDatautils.findDistrictById(commid)
if (dist) {
def key = dist.city+"#"+dist.region+"#"+dist.name
if(keys.contains(key)){
continue
}
keys << key
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100)
if(jsonObject && jsonObject.result){
def results = jsonObject.result
for(def r: results){
def row = []
row << d['B']
row << d['C']
row << d['D']
row << r.matchName
row << r.address
row << r.houseNum
row << r.roomNum
row << r.area
if(r.startDate){
row << r.startDate.substring(0, r.startDate.indexOf("T"))
}else{
row << ""
}
row << r.count
row << r.status
row << r.tradeAvgPrice
row << r.tradePrice
row << r.id
rows << row
}
println dist.name +" "+results.size()
}
}
}
}
ExcelUtils.write(output, rows, "拍卖案例")
}
def createHouse(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid = d['R']
// println "[createHouse] commid:"+commid
if (commid) {
def key = d['Q']
if (keys.contains(key)) {
continue
}
keys << key
def jsonObject = OpenApiClient.houseList(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "挂牌"
row << r.districtName
if(r.estateType){
row << r.estateType
}else{
if(r.purpose){
row << r.purpose
}else{
row << ""
}
}
if(r.room){
def roomStr = r.room+"室"
if(r.hall){
roomStr = roomStr +r.hall+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.area
row << r.fitment
row << r.pDate
row << ""
row << r.avgPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.height){
row << r.height
}else{
row << ""
}
row << r.direction
row << SourceNameUtil.getResourceName(r.root)
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "挂牌案例")
}
def createEvaluation(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid= d['R']
println "[createEvaluation] commid:"+commid
if (commid) {
if (keys.contains(commid)) {
continue
}
keys << commid
def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "评估"
row << r.commName
row << ""
if(r.roomcnt){
def roomStr = r.roomcnt+"室"
if(r.livingroomcnt){
roomStr = roomStr +r.livingroomcnt+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.buildArea
row << r.fitment
row << r.cDate.substring(0, r.cDate.indexOf("T"))
row << ""
row << r.unitPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.floorcnt){
row << r.floorcnt
}else{
row << ""
}
if(r.houseaspect && !r.houseaspect.contains("暂")){
row << r.houseaspect
}else{
row << ""
}
row << "房价网"
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "评估案例")
}
def createDealPrice(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid= d['R']
println "[createDealPrice] commid:"+commid
if (commid) {
if (keys.contains(commid)) {
continue
}
keys << commid
def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "成交"
row << r.districtName
if(r.type){
row << r.type
}else{
row << ""
}
if(r.room){
def roomStr = r.room+"室"
if(r.hall){
roomStr = roomStr +r.hall+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.area
row << r.fitment
row << r.tradeDate.substring(0, r.tradeDate.indexOf("T"))
row << ""
row << r.avgPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.height){
row << r.height
}else{
row << ""
}
if(r.direction && !r.direction.contains("暂")){
row << r.direction
}else{
row << ""
}
row << r.source
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "成交案例")
}
def addAuthor(def path){
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['X'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -68,6 +68,10 @@ class CommonDatautils {
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
public static DSCollection getHouseDataStoreConnection(){ public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) { if(house_dataStoreConnection != null) {
return house_dataStoreConnection return house_dataStoreConnection
@@ -586,6 +590,23 @@ class CommonDatautils {
} }
public static def findDistrictCount(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
q.put("region", region);
}
q.put("name", name);
q.put("cat", cat)
q.put("status", 0)
def count = getDataStoreConnection().count(q);
return count;
}
public static def findDistrict(String city, String region, String name, String cat){ public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
@@ -609,6 +630,7 @@ class CommonDatautils {
} }
public static def findDistrictList(String city, String region, String name, String cat){ public static def findDistrictList(String city, String region, String name, String cat){
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
@@ -704,7 +726,7 @@ class CommonDatautils {
q.put("visible",0) q.put("visible",0)
q.put("cat", cat); q.put("cat", cat);
q.put("city", city); q.put("city", city);
// q.put("name","吴中路9弄小区") // q.put("name","É­Íû·¼²ÝÂ̰¶")
DBObject keys = new BasicDBObject(); DBObject keys = new BasicDBObject();
if(fields){ if(fields){
@@ -827,16 +849,24 @@ class CommonDatautils {
keys.put("out_pics", 1); keys.put("out_pics", 1);
keys.put("lat", 1); keys.put("lat", 1);
keys.put("lng", 1); keys.put("lng", 1);
keys.put("lat2", 1);
keys.put("lng2", 1);
keys.put("name_cat", 1); keys.put("name_cat", 1);
keys.put("finish_building_date", 1) keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1) keys.put("mapping_alias", 1)
keys.put("sid", 1) keys.put("sid", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys); try{
if(cursor){ DSCursor cursor = getDataStoreConnection().find(q, keys);
for(def obj : cursor){ if(cursor){
return obj; for(def obj : cursor){
return obj;
}
} }
}catch(Exception e){
println "error_id:"+id
e.printStackTrace()
} }
return null; return null;
} }
@@ -1053,9 +1083,16 @@ class CommonDatautils {
public static def queryDistrictById(def id){ public static def queryDistrictById(def id){
def dbo = null
DBObject q = new BasicDBObject(); DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id)); try{
def dbo = getDataStoreConnection().findOne(q); q.put("_id", new ObjectId(id.trim().toString()));
dbo = getDataStoreConnection().findOne(q);
}catch(Exception e){
println id
}
return dbo; return dbo;
} }
@@ -2385,6 +2422,7 @@ class CommonDatautils {
q.put("block", block); q.put("block", block);
} }
q.put("cat", cat); q.put("cat", cat);
q.put("status",0)
def cur = getDataStoreConnection().find(q); def cur = getDataStoreConnection().find(q);
return cur.count(); return cur.count();
} }
@@ -3041,18 +3079,26 @@ class CommonDatautils {
queryCondition.put("region", region); queryCondition.put("region", region);
queryCondition.put("district_name", name); queryCondition.put("district_name", name);
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate)); queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate));
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)).limit(1) DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
DBObject dbo; DBObject dbo;
def prices = []
def index = 0
while(cur.hasNext()) { while(cur.hasNext()) {
dbo = cur.next() def obj = cur.next()
def price = dbo.get("avg_price"); if(index == 0){
if(price){ dbo = obj
break;
} }
prices << obj.get("avg_price");
index++
} }
return dbo; if(prices){
def avg_price = (int)(prices.sum()/prices.size())
dbo.avg_price = avg_price
return dbo;
}
return null;
} }
@@ -3185,4 +3231,18 @@ class CommonDatautils {
return avg_radios; return avg_radios;
} }
static def findLoudong(commid){
def q = new BasicDBObject()
q.put("id_in_seaweed", commid)
def count = LOUDONG_LINE_COLL.count(q)
return count;
}
static def findHouseLoudong(commid){
def q = new BasicDBObject()
q.put("district_id", commid)
def count = HOUSE_LOUDONG_COLL.count(q)
return count;
}
} }
@@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch {
@Override @Override
public void process(Object args) { public void process(Object args) {
def cities = ["北京", "上海"] def cities = ["北京", "上海"]//
for (def city : cities) { for (def city : cities) {
def ret = CommonDatautils.getAllDistricts("district", city) def ret = CommonDatautils.getAllDistricts("district", city)
// def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district"); // def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district");
@@ -6,10 +6,16 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jusfoun.JusfounCommon import com.fangjia.script.jusfoun.JusfounCommon
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
/** /**
@@ -19,8 +25,11 @@ import java.util.concurrent.TimeUnit
class DistrictPriceExportBatch extends BaseBatch{ class DistrictPriceExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def charset = "GBK" def charset = "GBK"
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) { static main(args) {
runBatch(new DistrictPriceExportBatch(), args) runBatch(new DistrictPriceExportBatch(), args)
} }
@@ -32,6 +41,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createDatas("¸£ÖÝ") def ret = createDatas("¸£ÖÝ")
saveToDB(ret)
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv" def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512" def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
@@ -76,10 +88,10 @@ class DistrictPriceExportBatch extends BaseBatch{
def createDatas(def city){ def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null); def districts = CommonDatautils.findAllDistricts(city, "district", null);
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def lastDate = new Date() def dataStr = DateUtils.add(endDateStr, -30)
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1) def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
def lastDate = DateUtils.parseFromTightByPattern(endDateStr, "yyyy-MM-dd")
// List<String> list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202111.csv")) // List<String> list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202111.csv"))
// def map2 = [:] // def map2 = [:]
// for(def s : list2) { // for(def s : list2) {
@@ -88,17 +100,34 @@ class DistrictPriceExportBatch extends BaseBatch{
// map2.put(key, row[5]) // map2.put(key, row[5])
// } // }
def lastMonthPriceMap = findFzDistrictStat("¸£ÖÝ",lastMonthDate)
def result = [] def result = []
for(def rMap : districts){ for(def rMap : districts){
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
if(dbo){ if(dbo){
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
def curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
if(markup < 0){
markup = randMarkup*(-1)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
def m = [] def m = []
m << dbo.get("_id").toString()//¾ù¼ÛID m << dbo.get("_id").toString()//¾ù¼ÛID
m << rMap.get("_id").toString()//Â¥ÅÌID m << rMap.get("_id").toString()//Â¥ÅÌID
m << rMap.city m << rMap.city
m << rMap.region m << rMap.region
m << rMap.name m << rMap.name
m << dbo.get("avg_price") m << curPrice
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name) // def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
// if(lastMonthPrice && dbo.get("avg_price")){ // if(lastMonthPrice && dbo.get("avg_price")){
// lastMonthPrice = lastMonthPrice.toInteger() // lastMonthPrice = lastMonthPrice.toInteger()
@@ -125,6 +154,19 @@ class DistrictPriceExportBatch extends BaseBatch{
return result return result
} }
def findFzDistrictStat(city,lastMonthDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("month", lastMonthDate)
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
def priceMap = [:]
for(def d : dbcursor){
priceMap.put(d.comm_id, d.avg_price)
}
return priceMap
}
def export(def fileName, def datas){ def export(def fileName, def datas){
def headers = ["id","comm_id","city","district","name","avg_price","year_month"] def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
writeCSV(headers, fileName, datas); writeCSV(headers, fileName, datas);
@@ -152,5 +194,44 @@ class DistrictPriceExportBatch extends BaseBatch{
file.append(System.getProperty("line.separator")); file.append(System.getProperty("line.separator"));
} }
} }
//Ëæ»ú¸¡µãÊý
public static double nextDouble(final double min, final double max) throws Exception {
DecimalFormat df = new DecimalFormat("#.00");
if (max < min) {
throw new Exception("min < max");
}
if (min == max) {
return min;
}
return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
}
def saveToDB(rets){
for(def r : rets){
def q = new BasicDBObject()
q.put("month", curDate.toInteger())
q.put("comm_id", r[1])
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("avg_price", r[5])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
def o = new BasicDBObject()
o.put("_id", new ObjectId(r[0]))
o.put("comm_id", r[1])
o.put("city", r[2])
o.put("region", r[3])
o.put("name", r[4])
o.put("avg_price", r[5].toInteger())
o.put("month", r[6].toInteger())
FZ_DISTRICT_STAT_COLL.save(o)
}
}
}
} }
@@ -8,7 +8,9 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBCursor import com.mongodb.DBCursor
import com.mongodb.DBObject import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import org.apache.commons.io.FileUtils import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
/** /**
* 福州数据结果统计 * 福州数据结果统计
@@ -17,16 +19,42 @@ class FzDataResultStat {
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) { static main(args) {
def task = new FzDataResultStat() def task = new FzDataResultStat()
task.priceMarkupRun(); task.priceMarkupRun();
task.regionPriceTrendRun() task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
def margeExcel(){
def list2= ExcelUtils.read("D:\\数据\\福州\\价格比对_202204_新2.xlsx", 0 ,1)
def map2 = [:]
for(def s : list2){
def key = s["A"]+"##"+s["B"]+"##"+s["C"]
map2.put(key, s["D"])
}
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\价格比对_202204_新3.xlsx",{row->
if(row["A"]){
def key = row["A"]+"##"+row["B"]+"##"+row["C"]
def price = map2.get(key)
if(price){
row['L'] = price
}
}
})
} }
def priceMarkupRun(){ def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:] def map1 = [:]
for(def s : list1){ for(def s : list1){
def row = s.split("@\\|@") def row = s.split("@\\|@")
@@ -137,16 +165,13 @@ class FzDataResultStat {
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate)); queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0; def prices = [];
while(cur.hasNext()) { while(cur.hasNext()) {
DBObject dbo = cur.next() DBObject dbo = cur.next()
price = dbo.get("avg_price"); prices << dbo.get("avg_price");
if(price){
break;
}
} }
def avg_price = (int)(prices.sum()/prices.size())
return price; return avg_price;
} }
@@ -165,4 +190,45 @@ class FzDataResultStat {
weekends = weekends.reverse(); weekends = weekends.reverse();
return weekends; return weekends;
} }
def saveCsvData(){
def files = getAllFiles("C:\\Users\\gaojuan\\福州数据(1)")
if(files){
for(def filepath : files){
List<String> list1= FileUtils.readLines(filepath)
println filepath
for(def s : list1){
def row = s.split("@\\|@")
def o = new BasicDBObject()
o.put("_id", new ObjectId(row[0]))
o.put("comm_id", row[1])
o.put("city", row[2])
o.put("region", row[3])
o.put("name", row[4])
o.put("avg_price", row[5].toInteger())
o.put("month", row[6].toInteger())
FZ_DISTRICT_STAT_COLL.save(o)
}
}
}
}
def static getAllFiles(def path) {
File file = new File(path);
File[] tempList = file.listFiles();
def files = [];
for (int i = 0; i < tempList.length; i++) {
if (tempList[i].isFile()) {
files << tempList[i];
}
if (tempList[i].isDirectory()) {
System.out.println("文件夹:" + tempList[i]);
}
}
return files;
}
} }
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息 //福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202203.csv") def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv")
def count0 = 0 def count0 = 0
def count1 = 0 def count1 = 0
def count2 = 0 def count2 = 0
@@ -112,22 +112,28 @@ class TradeExportBatch extends BaseBatch{
// q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate)) // q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate))
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000); def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000);
def rets = [] def rets = []
def keys = []
if (cursor != null) { if (cursor != null) {
while (cursor.hasNext()) { while (cursor.hasNext()) {
DBObject dbo = cursor.next(); DBObject dbo = cursor.next();
def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name); def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name);
if(district){ if(district){
def ret = []; def tradeDateStr = DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
ret << dbo._id.toString() def key = district._id.toString() +"##" + dbo.avg_price +"##"+dbo.area+"##"+tradeDateStr
ret << district._id.toString() if(!keys.contains(key)){
ret << dbo.city keys << key
ret << dbo.region def ret = [];
ret << dbo.district_name ret << dbo._id.toString()
ret << dbo.avg_price ret << district._id.toString()
ret << dbo.area ret << dbo.city
ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd") ret << dbo.region
ret << dbo.total_price ret << dbo.district_name
rets << ret; ret << dbo.avg_price
ret << dbo.area
ret << tradeDateStr
ret << dbo.total_price
rets << ret;
}
} }
} }
println "³É½»×ܼƣº"+rets.size() println "³É½»×ܼƣº"+rets.size()
@@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{
def run(){ def run(){
def cityMap = CommonDatautils.getAllCityRegions() def cityMap = CommonDatautils.getAllCityRegions()
cityMap.each{city,regions-> cityMap.each{city,regions->
findPriceByCity(city, regions);
findPriceByCity(city, regions);
} }
} }
@@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{
q.put("region" , r.region) q.put("region" , r.region)
q.put("month", r.month) q.put("month", r.month)
def mom = null def mom = null
if(lastObj){ if(lastObj && lastObj.avg_price && r.price){
mom = (r.price -lastObj.avg_price)/lastObj.avg_price mom = (r.price -lastObj.avg_price)/lastObj.avg_price
mom = DataHandleUtils.getDoubleRound(4, mom) mom = DataHandleUtils.getDoubleRound(4, mom)
} }
if(obj){ if(obj){
if(obj.avg_price != r.price){ if(obj.avg_price != r.price){
@@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{
def findEsAvgPrice(city, region){ def findEsAvgPrice(city, region){
def detail = [:] def detail = [:]
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1]) detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0])
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]]) detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
def must = [] def must = []
@@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{
if(!priceValue){ if(!priceValue){
priceValue = row['1-metric'] priceValue = row['1-metric']
} }
if(priceValue){ if(priceValue && priceValue.value){
price = priceValue.value.toInteger() price = priceValue.value.toInteger()
} }
result.price = price; result.price = price;
def date = row.key_as_string.replace("-","").substring(0, 6) def date = row.key_as_string.replace("-","").substring(0, 6)
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM") def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
if(date!=curMonth){ if(date!=curMonth){
result.month = date.toInteger() if(date.toInteger()>200001 && result.price){
rets << result result.month = date.toInteger()
rets << result
}
} }
} }
} }
@@ -164,6 +164,7 @@ class OpenApiUtils {
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){ if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null; return null;
} }
address = address.trim().replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all" String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
List<com.alibaba.fastjson.JSONObject> poiList = null; List<com.alibaba.fastjson.JSONObject> poiList = null;
@@ -18,44 +18,62 @@ class CcbCityPriceStat {
task.run() task.run()
} }
def run(){ def run(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
def rets = [] def rets = []
def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")
for(def city : citys){ for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"]) def matchTotalCount = findMatchTradeCount(city, startDate, endDate)
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30")) println city +" "+matchTotalCount
def cityRets = [] if(matchTotalCount){
for(def dist : districts){ def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01") def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29"))
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name) def cityRets = []
if(tradeAvgPrice && weekendPrice){ for(def dist : districts){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()] def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01")
cityRets << row def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
println row.toString() if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
}
} }
print city +" "+ cityRets.size()
rets.addAll(cityRets)
} }
print city +" "+ cityRets.size()
rets.addAll(cityRets)
} }
println "×ÜÁ¿£º"+rets.size() println "×ÜÁ¿£º"+rets.size()
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有") ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有")
}
def findMatchTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def count = DEAL_PRICE_COLL.count(q)
return count
} }
def findTrades(dist, startDate, endDate){ def findTrades(dist, startDate, endDate){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", dist.city) // q.put("city", dist.city)
q.put("region", dist.region) // q.put("region", dist.region)
//
if(dist.alias){ // if(dist.alias){
if(!dist.alias.contains(dist.name)){ // if(!dist.alias.contains(dist.name)){
dist.alias << dist.name // dist.alias << dist.name
} // }
q.put("district_name", new BasicDBObject('$in', dist.alias)) // q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{ // }else{
q.put("district_name", dist.name) // q.put("district_name", dist.name)
} // }
q.put("match_id", dist._id.toString())
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q) def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0 def avg_price = 0
@@ -199,18 +199,18 @@ class CcbCodeUtils {
static def getBuildStructureCode(def buildStructure){ static def getBuildStructureCode(def buildStructure){
if(buildStructure){ if(buildStructure){
if(buildStructure.contains("钢结构")){ if(buildStructure.contains("钢结构")){
return "001"; return "3";
}else if(buildStructure.contains("钢、钢筋混凝土结构")){ }else if(buildStructure.contains("钢、钢筋混凝土结构")){
return "002"; return "1";
}else if(buildStructure.contains("钢筋混凝土结构")){ }else if(buildStructure.contains("钢筋混凝土结构")){
return "003"; return "1";
}else if(buildStructure.contains("砖木结构")){ }else if(buildStructure.contains("砖木结构")){
return "004"; return "10";
}else if(buildStructure.contains("混合结构")){ }else if(buildStructure.contains("混合结构")){
return "005"; return "2";
} }
} }
return "006"; return "11";
} }
static def getDistHouseTypeCode(def estateType){ static def getDistHouseTypeCode(def estateType){
@@ -50,4 +50,32 @@ class FileUtils {
} }
out.close(); out.close();
} }
static def createTxtFile(ret, fileName, charset, isAppend){
File file = new File(fileName);
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(row)
sb.append("\n");
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
} }
@@ -0,0 +1,257 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeAppCrawl {
public static final SHORT_DATE = Utils.getLastMonth();
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
public static def SOURCE = "安居客";
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeAppCrawl();
// task.run();
// task.run2()
task.addThirdPrice()
}
def run2(){
def url = "https://m.anjuke.com/sh/community/"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def links = getRegion(url, driver)
for(def link : links){
def doc = getDocument(url, driver)
def str = doc.toString().substring(doc.toString().indexOf("<script> window.__NUXT__"), doc.toString().indexOf("</script>"))
if(str){
str = str.substring(str.indexOf("} ("), str.indexOf("));"))
def arrs = str.split(",")
pritln arrs.size()
}
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def getRegion(url, driver){
//def doc = getDocument(url, driver)
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/fz/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
header.put("sw8-correlation","c291cmNl:YmVpZG91")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
def doc = ABuYunUtil.doGet(url, null, header, "utf-8")
def links = []
if(doc){
def eles = JQueryUtils.select(".column2 a", doc)
for(def e : eles){
def href = e.attr("href")
links << href
}
}
return links;
}
def run(){
//上海:11,北京:14
//"仓山":4071,
// def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
// def regionMap = ["浦东":7,"闵行":11, "松江":15,"宝山":13,"嘉定":14,"徐汇":5,"青浦":19,"静安":2,"普陀":10,"杨浦":9,"奉贤":17,"黄浦":4,"虹口":8,"长宁":6,"金山":18,"崇明":20]
def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678]
def cityMap = ["北京":regionMap]
def cityId = 14
regionMap.each{region, area_id->
for(def i=1;i<40;i++){
def stop = false;
def all_stop = false
def error_count=0
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
//城口
def flagMap = getInfo(stop, i, area_id, cityId, "上海")
stop = flagMap.stop
all_stop = flagMap.all_stop
error_count++
if(error_count>=20){
stop = true
all_stop = true
}
}
if(all_stop){
break;
}
}
}
}
def getInfo(def stop, def i, def area_id, def cityId, def city){
def map = [:]
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/sh/community/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
header.put("sw8-correlation","c291cmNl:YmVpZG91")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
def all_stop = false;
def error_count = 0
if (ret) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if(jsonObj && jsonObj.status=="ok" ){
def data = jsonObj.data
if(data && data.total.toInteger()>0){
def communities = data.communities
println "【${i}】 "+communities.size() +" "+url
if(communities.size()>0){
save(city, communities)
Thread.sleep(10000)
stop = true;
}else{
stop = true;
println url +" 返回0"
all_stop = true;
}
}else{
stop = false;
error_count ++
println url +" 无数据"
}
}else{
stop = false;
Thread.sleep(5000)
}
}
map.stop = stop
map.all_stop = all_stop;
return map;
}
def save(city, def ret){
def index = 0
for(def o : ret){
index++
def obj = findIsExist(o.jumpAction)
if(!obj){
def udate = new BasicDBObject()
o.each{k,v->
udate.put(k,v)
}
udate.put("city", city)
udate.put("c_date", new Date())
detail.save(udate)
println index+" [保存]"+o.jumpAction
}else{
println index+" [已存在]"+o.jumpAction
}
}
}
def findIsExist(def url){
def q = new BasicDBObject()
q.put("jumpAction",url)
return detail.findOne(q);
}
def addThirdPrice(){
def q = new BasicDBObject()
def dbcursor = detail.find(q)
def index = 0
for(def d : dbcursor){
index++
def city = d.base.cityName
def region = d.base.areaName
def name = d.base.name
def obj = findByParams(city, region, name);
if(!obj){
def data = new BasicDBObject()
data.put("district_id",d.base.id)
data.put("city", city)
data.put("region", region )
data.put("name", name)
data.put("address",d.base.address)
data.put("avg_price", d.priceInfo.price)
data.put("source", SOURCE)
data.put("s_date", SHORT_DATE)
data.put("c_date", d.c_date)
thirdPartyColl.save(data)
println index +" "+data
}
}
}
def findByParams(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", SOURCE)
q.put("s_date", SHORT_DATE)
return thirdPartyColl.findOne(q);
}
}
@@ -11,9 +11,15 @@ import org.openqa.selenium.By
import org.openqa.selenium.WebDriver import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.remote.DesiredCapabilities
import org.openqa.selenium.remote.RemoteWebDriver
class AnjukeDetailCrawl { class AnjukeDetailCrawl {
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail") DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url")
static main(args) { static main(args) {
def task = new AnjukeDetailCrawl(); def task = new AnjukeDetailCrawl();
task.run(); task.run();
@@ -21,9 +27,15 @@ class AnjukeDetailCrawl {
def run(){ def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost"); System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://chongqing.anjuke.com/community/?from=navigation" def url = "https://chongqing.anjuke.com/"
WebDriver driver = new ChromeDriver() WebDriver driver = new ChromeDriver()
// DesiredCapabilities capabilities = DesiredCapabilities.chrome();
// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL
// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities);
driver.manage().deleteAllCookies()
def regions = getRegions(driver, url) def regions = getRegions(driver, url)
println "区域链接: "+regions.size()
getAllRegions(driver, regions) getAllRegions(driver, regions)
} }
@@ -31,44 +43,88 @@ class AnjukeDetailCrawl {
def getAllRegions(driver, regions){ def getAllRegions(driver, regions){
for(def url : regions){ for(def url : regions){
driver.get(url) def doc = getDocument(url, driver)
def doc = Jsoup.parse(driver.getPageSource());
def total = getTotal(doc) def total = getTotal(doc)
if(total) { if(total) {
if (total > 1250) { if (total > 1250) {
def links = getMoreLinks(url, driver)
println url +" "+links.size()
saveDetailLinks(links)
// parse(links,driver)
} else { } else {
def links = getAllLinks(url, driver) def links = getAllLinks(url, driver)
parse(links,driver) println url +" "+links.size()
saveDetailLinks(links)
// parse(links,driver)
} }
} }
} }
} }
def saveDetailLinks(links){
for(def url : links){
def data = new BasicDBObject()
data.put("city","重庆")
data.put("url", url)
data.put("source", "安居客")
data.put("c_date", new Date())
urlColl.save(data)
}
}
def getMoreLinks(url, driver){
def doc = Jsoup.parse(driver.getPageSource());
def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc)
def links = []
if(pricelinks){
for(def link : pricelinks){
if(link.text()!="不限"){
def urlLink = link.attr("href")
getDocument(urlLink, driver)
// driver.get(urlLink)
links.addAll(getAllLinks(urlLink, driver))
println urlLink +" "+links.size()
}
}
}
return links;
}
def getAllLinks(url, driver){ def getAllLinks(url, driver){
def stop = false def stop = false
def urls = [] def urls = []
while(!stop) { def doc = Jsoup.parse(driver.getPageSource());
def doc = Jsoup.parse(driver.getPageSource()); def total = getTotal(doc)
def page = getPage(total, 25)
def pages = []
for(def i=1;i<=page;i++){
pages << url.substring(0, url.size()-1)+"-p"+i+"/"
}
for(def pageUrl : pages){
doc = getDocument(pageUrl, driver)
def details = JQueryUtils.select(".list-cell a.li-row", doc) def details = JQueryUtils.select(".list-cell a.li-row", doc)
for (def d : details) { for (def d : details) {
urls << d.attr("href") def link = d.attr("href")
} if(!urls.contains(link)){
def nextEle = JQueryUtils.select(".next", doc) urls << d.attr("href")
if (nextEle) { }
driver.findElement(By.cssSelector(".next")).click()
}else{
stop = true
}
if(urls.size()>20){
stop = true
} }
} }
return urls return urls
} }
def getPage(def total, def pageSize){
if(total && total.toInteger()>0){
def page = total.toInteger()/pageSize;
if(page>(int)page){
page = (int)page + 1
return page
}
return (int)page;
}
return null
}
def getTotal(doc){ def getTotal(doc){
def totalText = JQueryUtils.selectText(".total-info", doc) def totalText = JQueryUtils.selectText(".total-info", doc)
@@ -82,9 +138,19 @@ class AnjukeDetailCrawl {
} }
def getRegions(driver, url){ def getRegions(driver, url){
// def doc = getDocument(url, driver)
driver.get(url) driver.get(url)
Thread.sleep(3000)
def doc = Jsoup.parse(driver.getPageSource()); def doc = Jsoup.parse(driver.getPageSource());
def els = JQueryUtils.select("a:contains(小区)", doc)
if(els){
def href = els[0].attr("href")
doc = Jsoup.parse(driver.getPageSource());
// driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click()
// driver.findElement(By.linkText("小区")).click()
doc = getDocument(href, driver)
}
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc) // def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
// if(loginIfm){ // if(loginIfm){
// def href = loginIfm[0].attr("src") // def href = loginIfm[0].attr("src")
@@ -107,23 +173,35 @@ class AnjukeDetailCrawl {
def regionEles = JQueryUtils.select(".region-parents li a", doc) def regionEles = JQueryUtils.select(".region-parents li a", doc)
def urls = [] def urls = []
for(def e : regionEles){ for(def e : regionEles){
urls << e.attr("href") if(e.text()!="不限"){
urls << e.attr("href")
}
} }
return urls; return urls;
} }
def getNontCheckPage(url, driver, doc){
def stop = false;
while(!stop){
if(doc){
}
}
return doc;
}
def parse(def links, driver){ def parse(def links, driver){
def rets = [] def rets = []
for(def link : links){ for(def link : links){
def district = parsing(link, driver) def district = parsing(link, driver)
if(district){ if(district){
rets << district // rets << district
if (rets.size() % 100 == 0) { // if (rets.size() % 100 == 0) {
def add= saveData(rets) saveObj(district)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
rets = [] // rets = []
} // }
} }
} }
if (rets.size()> 0) { if (rets.size()> 0) {
@@ -136,33 +214,61 @@ class AnjukeDetailCrawl {
def saveData(def results){ def saveData(def results){
def add = 0 def add = 0
for(def ret : results){ for(def ret : results){
def q = new BasicDBObject() saveObj(ret)
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
} }
return add
} }
def saveObj(def ret){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def parsing(url, driver){ def parsing(url, driver){
@@ -191,10 +297,9 @@ class AnjukeDetailCrawl {
def infoDiv = JQueryUtils.select(".info-list", doc) def infoDiv = JQueryUtils.select(".info-list", doc)
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text()) district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text()) district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text()) district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text()) district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text() def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
if (houseCountStr) { if (houseCountStr) {
houseCountStr = houseCountStr.replace("户", "") houseCountStr = houseCountStr.replace("户", "")
@@ -202,9 +307,13 @@ class AnjukeDetailCrawl {
district.house_count = houseCountStr.toInteger() district.house_count = houseCountStr.toInteger()
} }
} }
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text()) district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text())
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text()) def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
if(carSiteStr){ if(carSiteStr){
if(carSiteStr.contains("(")){ if(carSiteStr.contains("(")){
@@ -214,16 +323,19 @@ class AnjukeDetailCrawl {
district.car_site = carSiteStr district.car_site = carSiteStr
} }
} }
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text()) district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text()
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率:) .value").text()) district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text()) district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text()) district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
district.url = url district.url = url
district.c_date = new Date() district.c_date = new Date()
district.m_date = new Date() district.m_date = new Date()
district.crawl_type = "chrome"
district.root = "anjuke.com" district.root = "anjuke.com"
stop = true; stop = true;
}else{ }else{
Thread.sleep(3000) Thread.sleep(3000)
@@ -0,0 +1,89 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class AnjukeDistrictMatch {
static main(args) {
def task = new AnjukeDistrictMatch();
task.run();
// task.run1()
}
def run1(){
def regionPointsMap = findRegionRange("重庆")
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
}
def run(){
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
ExcelUtils.multiThreadingModify(filePath,{row->
if(row['A'] && !row['J']){
def city = row['B']
def region = row['C']
def name = row['D']
def address = row['E']
// def flag = "别名匹配"
// def dist = matchDist(city, null, name)
// if(!row['J'] && name.contains("(")){
// name = name.replace("(${region})","").replace("(","").replace(")","")
// flag = "去括号匹配"
// dist = matchDist(city, region, name)
// }
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
// flag = "映射别名匹配"
// }
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
// if(rets && rets.size()==1){
// dist = rets[0]
// }
// }
//
// if(!dist){
// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district")
// flag = "地址匹配"
// if(dists.size()>1){
// println address +" 地址相同:"+dists.size()
// }else{
// if(dists.size()==1){
// dist = dists[0]
// }
// }
// }
// if(dist){
// row['J'] = flag
// row['K'] = dist._id.toString()
// row['L'] = dist.region
// row['M'] = dist.name
// row['N'] = dist.address
// }
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
if(jsonObject != null && jsonObject.precise!=-1) {
// row['J'] = "匹配接口"
// row['K'] = jsonObject.get("id")
// row['L'] = jsonObject.get("district")
// row['M'] = jsonObject.get("name")
// row['N'] = jsonObject.get("address")
row['M'] = jsonObject.get("name")
}
}
})
}
def matchDist(city, region, name){
def dist = CommonDatautils.findDistrict(city, region, name, "district")
return dist;
}
}
@@ -0,0 +1,189 @@
package com.fangjia.script.gj.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeHouseCrawl {
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
static main(args) {
def task = new AnjukeHouseCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
for(def i=24;i<50;i++){
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
def doc = getDocument(url, driver)
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
println "页数:"+i+" 记录数:"+eles.size()
if(eles){
parser(eles, driver)
}
}
}
def parser(eles, driver){
def index = 0
for(def e : eles){
index ++
def href = e.attr("href")
def doc = getDocument(href, driver)
// parserHtml(doc)
def newUrl = href.substring(0, href.indexOf("?"))
sendMq(newUrl, doc)
println "[ "+index +"] "+ newUrl
}
}
def sendMq(href, doc){
JSON json = new JSONObject()
json.put("project", "sell_anjuke")
json.put("url", href)
JSON result = new JSONObject()
result.put("content", doc.toString())
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
}
def parserHtml(doc) {
String city = null
def result = [:]
result.siteCode = "安居客"
if (doc) {
result.title = JQueryUtils.selectText([".banner-title h1"], doc)
def heads = JQueryUtils.select(".crumbs a", doc)
if (heads && heads.size() >= 5) {
result.city = heads[0].text().replace("房产网", "")
def orgRegion = heads[2].text().replace("二手房", "")
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
result.block = heads[3].text().replace("二手房", "")
result.district_name = heads[4].text()
} else if (heads && heads.size() >= 3) {
result.city = heads[0].text().replace("房产网", "")
result.district_name = heads[2].text()
def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
if (eles && eles.size() > 0) {
def orgRegion = eles[0].text()
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
}
}
if (!result.district_name) {
println " [未解析出(城市-区域-名称)信息]"
}
println result
def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
if (priceStr) {
if (priceStr.isNumber()) {
result.total_price = priceStr.toDouble() * 10000
}
}
def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
if (roomInfoStr) {
Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
if (room) {
result.room = room
}
if (hall) {
result.hall = hall
}
if (toilet) {
result.toilet = toilet
}
}
def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
if (areaStr) {
result.area = areaStr.toDouble()
}
def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
result.direction = DataCrawlUtils.parseDirection(directionStr)
def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
if (buildDateStr && buildDateStr.contains("/")) {
def arrs = buildDateStr.split("\\/")
if (arrs && arrs.size() == 2) {
result.build_date = arrs[0]
result.estate_type = arrs[1]
}
}
result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
if (result.floor_info && result.floor_info.contains("共")) {
def height = result.floor_info
if (result.floor_info.contains("(")) {
height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
}
height = height.replace("共", "").replace("层", "")
if (height) {
result.height = height.toInteger()
}
}
if (result.area && result.total_price) {
result.avg_price = (result.total_price / result.area).toInteger()
}
def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(5000)
def els = JQueryUtils.select(".list-main .list-left .list", doc)
if(!els){
els = JQueryUtils.select(".props-main", doc)
}
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -0,0 +1,631 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
class AnjukeToDB {
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeToDB();
task.run();
// task.addNoMatch()
// task.updateEstateType()
// task.gaodeRangeData()
}
def updateEstateType(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1)
for(def row : datas){
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj) {
def type = getEstatType(obj.extend.type)
def commId = row['L']
if(!commId){
commId = row['G']
}
if(type){
if(type=="别墅"){
def udata = new BasicDBObject()
udata.put("estate_type2", type)
udata.put("m_date", new Date())
println commId
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata))
}
}else{
// def udata = new BasicDBObject()
// udata.put("status", 1)
// udata.put("name_cat",11)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata))
}
}
}
}
def gaodeRangeData(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1)
def index = 0
for(def d : datas){
index++
println index
def personName = d['AB']
if(personName){
def commid = d['J']
if(commid){
def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district")
if(obj){
updateDist(d, obj._id.toString())
}
}else{
saveGaode(d)
}
}else{
def commid = d['J']
updateDist(d, commid)
}
}
}
def saveGaode(d){
def oldName = d['E']
def distName = nameFileter(d['C'], oldName)
def udata = new BasicDBObject()
def alias = []
def aliasStr = d['F']
if(aliasStr){
def arrs = aliasStr.split("/")
for(def s : arrs){
alias << s
}
}
if(alias.size()>0){
udata.put("alias", alias)
}
def address = AddressUtil.formatAddress("重庆", d['G'])
if(address){
udata.put("address", address)
}
if(d['N']){
udata.put("virescence_percent", d['N'])
}
if(d['O']){
udata.put("estate_charge", d['O'])
}
if(d['Q']){
udata.put("cubage_percent", d['Q'])
}
if(d['R']){
udata.put("developer", d['R'])
}
if( d['S']){
udata.put("build_area", d['S'])
}
if(d['T']){
def t = d['T'].substring(0, 3)
udata.put("build_date", t)
}
if(d['V'] && d['V'].isNumber()){
udata.put("car_site_count", d['V'].toInteger())
}
if(d['Y']){
def build_type = getBuildType2(d['Y'])
if(build_type){
udata.put("build_type", build_type)
}
}
if(d['Z']){
udata.put("property_life", d['Z'])
}
def lat = null
def lng = null
if(d['H']){
lat = d['H']
lng = d['I']
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
Long dlat = lat.toDouble() * 10000000000L
Long dlng = lng.toDouble() * 10000000000L
udata.put("lat", dlat)
udata.put("lng", dlng)
}
def type = "普通住宅"
if(distName.contains("别墅")) {
type = "别墅"
}else if(distName.contains("商住")){
type = "商住"
}
if(udata){
udata.put("name_cat",0)
udata.put("status",0)
udata.put("cat","district")
udata.put("c_date", new Date())
udata.put("m_date", new Date())
udata.put("city", "重庆")
udata.put("region",d['C'])
udata.put("name", distName)
udata.put("estate_type2",type)
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map",markMap)
DbUtils.seaweed.save(udata);
def stop = false
while (!stop){
if(udata._id.toString()){
addPrice(udata)
stop = true;
}
}
}
}
def updateDist(d, commid){
if(commid){
def dist = CommonDatautils.queryDistrictById(commid)
if(dist){
def oldName = d['E']
def distName = nameFileter(d['C'], oldName)
def udata = new BasicDBObject()
if(!distName.contains("号")){
def alias = dist.alias
if(!alias){
alias = []
}
if(!alias.contains(distName) && dist.name!=distName){
alias << distName
}
def aliasStr = d['F']
if(aliasStr){
def arrs = aliasStr.split("/")
for(def s : arrs){
alias << s
}
}
if(alias.size()>0 && alias!=dist.alias){
udata.put("alias", alias)
}
}
if(dist.address && !dist.address.contains("号")){
def address = AddressUtil.formatAddress("重庆", d['G'])
if(address){
udata.put("address", address)
}
}
if(!dist.virescence_percent && d['N']){
udata.put("virescence_percent", d['N'])
}
if(!dist.estate_charge && d['O']){
udata.put("estate_charge", d['O'])
}
if(!dist.cubage_percent && d['Q']){
udata.put("cubage_percent", d['Q'])
}
if(!dist.developer && d['R']){
udata.put("developer", d['R'])
}
if(!dist.build_area && d['S']){
udata.put("build_area", d['S'])
}
if(!dist.build_date && d['T']){
if(d['T'].size()>=4){
def t = d['T'].substring(0, 4)
udata.put("build_date", t)
}
}
if(!dist.car_site_count && d['V'] && d['V'].isNumber()){
udata.put("car_site_count", d['V'].toInteger())
}
if(!dist.build_type && d['Y']){
def build_type = getBuildType2(d['Y'])
if(build_type){
udata.put("build_type", build_type)
}
}
if(!dist.property_life && d['Z']){
udata.put("property_life", d['Z'])
}
def lat = null
def lng = null
if(d['H']){
lat = d['H']
lng = d['I']
def slat = dist.lat/10000000000.0
def slng = dist.lng/10000000000.0
def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat);
if(distance>1000){
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
Long dlat = lat.toDouble() * 10000000000L
Long dlng = lng.toDouble() * 10000000000L
udata.put("lat", dlat)
udata.put("lng", dlng)
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改] ${dist.name}"+ udata
}
}
}
}
def addNoMatch(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1)
for(def d : datas){
add(d)
}
}
def run(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\非人工处理.xlsx",0,1)
for(def data : datas){
// if(data['K']=="镕杰"){
def flag = data['M']
if(flag && flag=="新增"){
// add(data)
}else{
update(data)
}
// }
}
}
def add(def row){
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj){
def type = getEstatType(obj.extend.type)
if(type){
def city = obj.base.cityName
def region = obj.base.areaName
def lat2 = null
def lng2 = null
def lat = null
def lng = null
if(obj.base.lat){
lat2 = obj.base.lat
lng2 = obj.base.lng
JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu")
if(convertObj.getInt("status") == 1) {
String lo = convertObj.get("locations")
lng = Double.parseDouble(lo.split(",")[0])
lat = Double.parseDouble(lo.split(",")[1])
}
}
def distName = nameFileter(obj.base.areaName, obj.base.name)
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
def address = AddressUtil.formatAddress(obj.base.city, obj.base.address)
def dist = CommonDatautils.findDistrict(city, region, distName, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", distName)
data.put("estate_type2",type)
if(address){
data.put("address", obj.base.address)
}
if(lat){
Long alat = lat*10000000000L
Long alng = lng*10000000000L
data.put("lat", alat)
data.put("lng", alng)
Long alat2 = lat2.toDouble()*10000000000L
Long alng2 = lng2.toDouble()*10000000000L
data.put("lat2",alat2)
data.put("lng2",alng2)
}
data.put("finish_building_date",obj.extend.completionTime)
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){
data.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
}
data.put("build_area", obj.extend.totalArea)
data.put("virescence_percent", obj.extend.landscapingRatio)
data.put("build_type", buildTypeStr)
data.put("business_quarter",obj.base.tradingAreaName)
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
DbUtils.seaweed.save(data);
def stop = false
while (!stop){
if(data._id.toString()){
addPrice(data)
stop = true;
}
}
}
}else{
println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type
}
}else{
println anjukeId+" 没找到"
}
}
def update(def row){
def commId = row['L']
if(!commId){
commId = row['G']
}
def dist = CommonDatautils.queryDistrictById(commId)
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj && dist){
def udata = new BasicDBObject()
def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address)
if(address){
if(!dist.address){
if(obj.base.address){
udata.put("address",obj.base.address)
}
}else{
if(dist.address.contains("号")){
if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){
def addr_alias = dist.addr_alias
if(!addr_alias){
addr_alias = []
}
if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){
addr_alias << obj.base.address
udata.put("addr_alias", addr_alias)
}
}
}else{
udata.put("address",obj.base.address)
}
}
}
def distName = nameFileter(obj.base.areaName, obj.base.name)
if(!distName.contains("号")){
def alias = dist.alias
if(!alias){
alias = []
}
if(alias.contains(distName)){
alias << distName
}
if(alias.size()>0 && alias!=dist.alias){
udata.put("alias", alias)
}
}
if(!dist.build_type){
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
if(buildTypeStr){
udata.put("build_type",buildTypeStr)
}
}
if(!dist.household_count){
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){
udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
}
}
if(!dist.finish_building_date){
if(obj.extend.completionTime){
udata.put("finish_building_date",obj.extend.completionTime)
}
}
if(!dist.virescence_percent){
if(obj.extend.landscapingRatio){
udata.put("virescence_percent", obj.extend.landscapingRatio)
}
}
if(!dist.business_quarter){
if(obj.base.tradingAreaName){
udata.put("business_quarter",obj.base.tradingAreaName)
}
}
def lat2 = null
def lng2 = null
if(dist.lat2){
lat2 = dist.lat2/10000000000.0
lng2 = dist.lng2/10000000000.0
}
if(dist.lat && !lat2){
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3)
lng2 = baidu.x
lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
}
if(obj.base.lat) {
def alat2 = obj.base.lat.toDouble()
def alng2 = obj.base.lng.toDouble()
def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2);
if(distance>1000){
println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................"
def lng = null
def lat = null
JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu")
if(convertObj.getInt("status") == 1) {
String lo = convertObj.get("locations")
lng = Double.parseDouble(lo.split(",")[0])
lat = Double.parseDouble(lo.split(",")[1])
}
Long blat2 = alat2 *10000000000L
Long blng2 = alng2 *10000000000L
Long gLat2 = lat *10000000000L
Long gLng2 = lng *10000000000L
udata.put("lat2", blat2)
udata.put("lng2", blng2)
udata.put("lat", gLat2)
udata.put("lng", gLng2)
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改] ${dist.name}"+ udata
}
}
}
def findAnjukeDistrictDetail(anjukeId){
def q = new BasicDBObject()
try{
q.put("_id",new ObjectId(anjukeId))
return detail.findOne(q)
}catch(Exception e){
print "[查询出错]anjukeId:"+anjukeId
}
return null
}
def nameFileter(def region, def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
name = name.replace("(${region})","").replace("(","").replace(")","")
return name;
}
def getEstatType(type){
if(type){
if(type=="公寓住宅"){
type = "普通住宅"
}else if(type.contains("商住")){
type = "商住"
}else if(type.contains("别墅")){
type = "别墅"
}else{
type = null
}
}
return type
}
def getBuildType(def buildType){
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
def list = []
if(buildType){
def arrs = buildType.split("\\|")
for(def s : arrs){
if(types.contains(s)){
list << s
}
}
if(list){
return list.join(",")
}
}
return null
}
def getBuildType2(def buildType){
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
def list = []
if(buildType){
def arrs = buildType.split(",| ")
for(def s : arrs){
if(types.contains(s)){
list << s
}
}
if(list){
return list.join(",")
}
}
return null
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
}
@@ -0,0 +1,250 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataCrawlUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
str = str.replace("元/m?", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
str = str.replace("m?", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
}
@@ -0,0 +1,347 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 成交数据解析工具类
*/
class DealPriceUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m²"), { String str ->
str = str.replace("元/m²", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m²"), { String str ->
str = str.replace("m²", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析成交日期
* @param content
* @return
*/
static Date parseDealDate(String content) {
return parse(content, DEAL_DATE_RESOLVERS)
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
/**
* 解析总价
* @param content
* @return
*/
static Integer parseTotalPrice(String content) {
return parse(content, TOTAL_PRICE_RESOLVERS)
}
/**
* 解析单价
* @param content
* @return
*/
static Integer parseUnitPrice(String content) {
return parse(content, UNIT_PRICE_RESOLVERS)
}
/**
* 解析面积
* @param content
* @return
*/
static Double parseArea(String content) {
return parse(content, AREA_RESOLVERS)
}
/**
* 解析楼层
* @param content
* @return
*/
static Integer parseFloor(String content) {
return parse(content, FLOOR_RESOLVERS)
}
/**
* 解析楼高
* @param content
* @return
*/
static Integer parseHeight(String content) {
return parse(content, HEIGHT_RESOLVERS)
}
/**
* 格式化小区名
* @param name
* @return
*/
static String formatName(String name) {
name = name.replace("-商住", "").replace("-别墅", "")
//汇福家园平安里(迁)2015-2017
name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "")
//国贸天悦一期2017
name = name.replaceAll("\\d\\d\\d\\d\$", "")
if (name ==~ /.+(\(|()[\d一二三四五六七八九十东南西北|\w]+(期|区)(\)|))$/) {
name = name.replaceAll("\\(|(|\\)|)", '')
} else if (name ==~ /.+(\(|()别墅(\)|))$/) {
name = name.replaceAll("\\(|(|\\)|)", '')
} else if (name ==~ /.+(\(|().+(\)|))$/) {
name = name.replaceAll("(\\(|().+(\\)|))", '')
} else if (name.contains("·") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) {
println("---------------------------->" + name)
name = name.replaceAll("·|-", '')
println(name)
}
if (name ==~ /.{2,}路.{3,}$/ && !(name ==~ /.{2,}路.+号/) && !(name ==~ /.{2,}路.+弄/) && !(name ==~ /.*小区$/)) {
name = name.replaceAll("^.{2,}路", "")
}
return name
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
/**
* 匹配符合条件的字符串
*
* @param str
* @param pattern
* @return
*/
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parsePrice(String priceStr) {
def pattern =Pattern.compile("\\d+(\\.\\d+)?万")
String match = match(priceStr, pattern)
}
static void main(String[] args) {
println parsePrice("顶秀欣园西苑2室1厅1卫492.3万")
}
}
@@ -0,0 +1,167 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class ZhugeFangDistCrawl {
public static final SHORT_DATE = Utils.getLastMonth();
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new ZhugeFangDistCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
WebDriver driver = new ChromeDriver()
driver.manage().deleteAllCookies()
getRegions(driver, url)
}
def getRegions(driver, url) {
for(def i=1;i<=11;i++){
def newUrl = url+"page${i}/"
println newUrl
def doc = getDocument(newUrl, driver)
def total = parseList(doc)
// def next = JQueryUtils.select("a:contains(下一页)", doc)
// if(next){
// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
// }
}
}
def parseList(doc){
def city= "福州"
def els = JQueryUtils.select("#listTableBox li", doc)
if (els) {
for(def e : els){
def name = e.select(".house-name").text()
def str = e.select(".house-adr").text()
def address = null
if(str){
if(str.contains("]")){
address = str.substring(str.indexOf("]")+1).trim()
}else{
address = str
}
}
def regionStr = null
if(str && str.contains("]") && str.contains("[")){
regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]"))
}
def region = null
def block = null
if(regionStr){
def arrs = regionStr.split("-")
if(arrs && arrs.size()>0){
if(arrs.size()==2){
block = arrs[1].trim()
}
if(arrs.size()>=1){
region = arrs[0].trim()
}
}
}
def links = e.select(".house-name a")
def href = null
if(links){
href = links[0].attr("href")
}
def priceStr = e.select(".price-number").text()
def price = null
if(priceStr && !priceStr.contains("暂无")){
if(priceStr.contains("元")){
priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim()
}
if(priceStr && priceStr.isNumber()){
price = priceStr.toInteger()
}
}
def obj = findByParams(city, region, name)
if(obj){
if(price){
DBObject udata2 = new BasicDBObject()
udata2.put("avg_price", price)
udata2.put("m_date", new Date())
println "[upeate] ${region} "+name
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
}
}else{
def udata = new BasicDBObject()
udata.put("city", city)
udata.put("region", region)
udata.put("block", block)
udata.put("url", href)
udata.put("address", address)
udata.put("name", name)
udata.put("avg_price", price)
udata.put("s_date", SHORT_DATE)
udata.put("c_date", new Date())
udata.put("m_date", new Date())
udata.put("source", "诸葛找房")
thirdPartyColl.save(udata)
println "[save] ${region} "+name+" "+priceStr
}
}
}
return els.size()
}
def findByParams(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", "诸葛找房")
q.put("s_date", SHORT_DATE)
return thirdPartyColl.findOne(q);
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(10000)
def els = JQueryUtils.select("#listTableBox li", doc)
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -172,7 +172,8 @@ class NameProcess {
if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){ if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){
return null return null
} }
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){ //|| name.endsWith("组")
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){
return null return null
} }
if(name.contains("、")||name.contains("|")||name.contains(".")){ if(name.contains("、")||name.contains("|")||name.contains(".")){
@@ -71,7 +71,8 @@ class DistrictIndexBatch extends BaseBatch{
def run(){ def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = CommonDatautils.findAllCity(); def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
//def citys = CommonDatautils.findAllCity();
for(def city : citys){ for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"]) def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
def code = cityCodeDatas.get(city) def code = cityCodeDatas.get(city)
@@ -86,7 +87,7 @@ class DistrictIndexBatch extends BaseBatch{
city_avg_total_price = 531.2615913 city_avg_total_price = 531.2615913
} }
if(city_avg_total_price){ if(city_avg_total_price){
// calcute(city, dists, code, city_avg_total_price) calcute(city, dists, code, city_avg_total_price)
calcuteIndex(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price)
} }
@@ -0,0 +1,123 @@
package com.fangjia.script.gj.location
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
class DistrictMapMatch {
static main(args) {
def task = new DistrictMapMatch()
task.mapMatch()
}
def mapMatch(){
def index = 0
ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d->
if(!d['H']){
index++
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// if (!map || !map.lat) {
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// }
if(map){
d['F'] = map.id
if(map.adname){
d['G'] = map.adname
}else if(map.region){
d['G'] = map.region
}
d['H'] = map.name
d['I'] = map.address
if(map.type){
d['J'] = map.type
}else if(map.estateType){
d['J'] = map.estateType
}
d['K'] = map.lng
d['L'] = map.lat
d['M'] = map.score
d['N'] = map.source
}
}
},0)
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -21,10 +21,58 @@ class MatchTest {
def task = new MatchTest() def task = new MatchTest()
// task.run() // task.run()
// task.addNum() // task.addNum()
task.dongguang() // task.dongguang()
// task.meituanDel() // task.meituanDel()
task.addLevel()
} }
def addLevel() {
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
ExcelUtils.multiThreadingModify(path, { row ->
def name = row['D']
def address = row['E']
def city = RegionData.formatCity(row['A'])
if(name && !row['O']){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
// if(rMap.level){
// row['Q'] = rMap.level
// }
}
}
}
})
}
def meituanDel(){ def meituanDel(){
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx" def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
def index = 0 def index = 0
@@ -122,7 +122,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href") def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text() def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){ if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl detailUrls << durl
}else{ }else{
@@ -153,7 +153,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href") def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text() def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){ if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl detailUrls << durl
parse(city, e) parse(city, e)
@@ -193,7 +193,7 @@ public class BeiKeCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
} }
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")) detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
} }
} }
@@ -215,7 +215,8 @@ public class BeiKeCrawlByChrome {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
} }
detail.source = "贝壳找房" detail.source = "贝壳找房"
def dbo = findTrade(detail) def detailUrl = e.select(".title a").attr("href")
def dbo = findTrade(detailUrl)
if (!dbo) { if (!dbo) {
def houseInfo = e.select(".houseInfo").text() def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|") def houseInfos = houseInfo.split("\\|")
@@ -228,7 +229,7 @@ public class BeiKeCrawlByChrome {
detail.height = floorStr.toInteger() detail.height = floorStr.toInteger()
} }
} }
detail.url = e.select(".title a").attr("href") detail.url = detailUrl
detail.create_date = new Date() detail.create_date = new Date()
detail.m_date = new Date() detail.m_date = new Date()
println detail println detail
@@ -313,8 +314,8 @@ public class BeiKeCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) { if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")) detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")) detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")).toInteger()
} }
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -336,13 +337,14 @@ public class BeiKeCrawlByChrome {
} }
def findTrade(def detail){ def findTrade(def detailUrl){
def q = new BasicDBObject() def q = new BasicDBObject()
q.put("city", detail.city) q.put("city", detail.city)
q.put("region", detail.region) // q.put("region", detail.region)
q.put("district_name", detail.district_name) // q.put("district_name", detail.district_name)
q.put("area", detail.area) // q.put("area", detail.area)
q.put("total_price", detail.total_price) // q.put("total_price", detail.total_price)
q.put("url", detailUrl)
q.put("trade_date", detail.trade_date) q.put("trade_date", detail.trade_date)
q.put("source", detail.source) q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q) def dbo = DEAL_PRICE_COLL.findOne(q)
@@ -358,13 +360,13 @@ public class BeiKeCrawlByChrome {
if(cityEles){ if(cityEles){
for(def e : cityEles){ for(def e : cityEles){
def href = e.attr("href") def href = e.attr("href")
if(e.text().contains("ÖØÇì")){ // if(e.text().contains("¸£ÖÝ")){
def cityMap = [:] def cityMap = [:]
cityMap.city = e.text() cityMap.city = e.text()
cityMap.url = "https:"+href cityMap.url = "https:"+href
list << cityMap list << cityMap
//cityMap.put(e.text(), "https:"+href) //cityMap.put(e.text(), "https:"+href)
} // }
} }
} }
return list return list
@@ -29,7 +29,7 @@ class JdPaimaiCrawl {
task.run(); task.run();
} }
def run(){ def run(){
def existsIds = findAllIds() def existsIds = []//findAllIds()
println "ÒÑ´æÔÚ:"+existsIds.size() println "ÒÑ´æÔÚ:"+existsIds.size()
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost"); System.setProperty("http.nonProxyHosts", "localhost");
@@ -43,78 +43,80 @@ class JdPaimaiCrawl {
def getList(driver){ def getList(driver){
def list = [] def list = []
for(def i=1;i<=32;i++){ list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268"
if(i==8){ // for(def i=1;i<=32;i++){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" // if(i==8){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
}else if(i==5){ // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" // }else if(i==5){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
}else if(i==17){ // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" // }else if(i==17){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
}else if(i==19){ // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" // }else if(i==19){
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
}else{ // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" // }else{
} // list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
} // }
// }
def links = [] def links = []
for(def url : list){ for(def url : list){
driver.get(url) driver.get(url)
@@ -430,10 +432,10 @@ class JdPaimaiCrawl {
def s1 = JQueryUtils.selectText(["li:contains(¾ºÂò¹«¸æ) .floor-content"], doc) def s1 = JQueryUtils.selectText(["li:contains(¾ºÂò¹«¸æ) .floor-content"], doc)
if(s1) { if(s1) {
def start = s1.indexOf("·¨Ôº½«ÓÚ") def start = s1.indexOf("将于")
def end = s1.indexOf("Ö¹") def end = s1.indexOf("Ö¹")
if(start>-1 && end>-1 && end>start){ if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("·¨Ôº½«ÓÚ"), s1.indexOf("Ö¹")) s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止"))
} }
def dateRangs = DataUtil.getDateRange(s1) def dateRangs = DataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) { if(dateRangs && dateRangs.size()==2) {
@@ -476,6 +478,11 @@ class JdPaimaiCrawl {
} }
} }
if(!result.startDate){
def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","")
result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日")
}
if(result.finishDate){ if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
} }
@@ -38,6 +38,26 @@ public class LianjiaCrawlByChrome {
// task.run1() // task.run1()
task.addDealRegion(); task.addDealRegion();
task.standardRegion() task.standardRegion()
// task.updadteDealPriceField()
}
def updadteDealPriceField(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def city : citys){
index ++
def q = new BasicDBObject()
q.put("city", city)
q.put("hall", "0")
def count = DEAL_PRICE_COLL.count(q)
println "[${index}] "+city +" "+count
if(count && count>0){
def udata = new BasicDBObject()
udata.put("hall", 0)
DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
println "[ÐÞ¸Ä]"+city +" "+ count
}
}
} }
def run1(){ def run1(){
@@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href") def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text() def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){ if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl detailUrls << durl
}else{ }else{
@@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href") def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text() def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-02-20", "yyyy-MM-dd") def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){ if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl detailUrls << durl
parse(city, e) parse(city, e)
@@ -258,7 +278,7 @@ public class LianjiaCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
} }
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")) detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
} }
} }
@@ -378,8 +398,8 @@ public class LianjiaCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) { if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")) detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")) detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")).toInteger()
} }
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -420,11 +440,11 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource()); def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:] def cityMap = [:]
def citys= ["³¤´º","¼ªÁÖ"] def citys= ["¸£ÖÝ"]
if(cityEles){ if(cityEles){
for(def e : cityEles) { for(def e : cityEles) {
def href = e.attr("href") def href = e.attr("href")
if (!href.contains("fang.lianjia") && citys.contains(e.text())){ if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href) cityMap.put(e.text(), href)
} }
} }
@@ -4,9 +4,12 @@ import com.alibaba.fastjson.JSON
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.chrome.ChromeDriver
@@ -19,9 +22,102 @@ class PaimaiTest {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
static main(args) { static main(args) {
def task = new PaimaiTest(); def task = new PaimaiTest();
task.run(); // task.run();
// task.stat()
task.updatePaimai()
// task.updateFieldPaimai()
}
def updateFieldPaimai(){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in',["重庆","重庆市"]))
def index = 0
def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
for(def obj : dbcursor) {
index++
println index
if(obj.address && obj.address.contains("可贷款")){
obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款")))
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
}
}
}
def updatePaimai(){
def q = new BasicDBObject()
q.put("url", Pattern.compile(".*jd.com.*"))
def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def index =0
for(def d : dbcursor){
index++
println index
if(!d.url.contains("jd.com")){
continue
}
if(!d.proince.contains("重庆")){
continue
}
println d.url
def query = new BasicDBObject()
query.put("url", d.url)
def dbo = paimai43_COLL.findOne(query);
DBObject obj = new BasicDBObject();
d.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimai43_COLL.save(obj)
}catch(Exception e){
println obj.url
}
}
}
}
def stat(){
def q = new BasicDBObject()
q.put("city", new BasicDBObject('$in', ["重庆","重庆市"]))
def dbcursor = paimaiResultColl.find(q)
def keys = []
for(def db : dbcursor){
if(db.matchCity){
def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName;
if(!keys.contains(key)){
keys << key
}
// try{
// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null)
// if(jsonObject != null && jsonObject.precise!=-1) {
// db.matchCity = jsonObject.get("city")
// db.matchRegion = jsonObject.get("district")
// db.matchName = jsonObject.get("name")
// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db))
// println db.address+" "+db.matchName
// }
// }catch(Exception e){
// println "error "+db.address
// }
}
}
println keys.size()
} }
def run(){ def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
@@ -11,14 +11,17 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId import org.bson.types.ObjectId
import java.text.DecimalFormat import java.text.DecimalFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher import java.util.regex.Matcher
import java.util.regex.Pattern import java.util.regex.Pattern
@@ -34,6 +37,9 @@ class CZDataBatch extends BaseBatch{
DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com") DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com")
//final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls"); //final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls");
//final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls"); //final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls");
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) { static main(args) {
runBatch(new CZDataBatch(), args); runBatch(new CZDataBatch(), args);
} }
@@ -41,11 +47,17 @@ class CZDataBatch extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
def citys = CommonDatautils.findAllCity(); def citys = CommonDatautils.findAllCity();
citys.remove("常州"); citys.remove("常州");
citys.add(0, "常州") citys.add(0, "常州")
for(def city : citys){ for(def city : citys){
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend)
def commids = findAllJnBankDatas(city)
def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields));
districtRun(districts); districtRun(districts, statMap, commids);
def shops = CommonDatautils.findAllDistricts(city, "shop", fields); def shops = CommonDatautils.findAllDistricts(city, "shop", fields);
shops = mergeShop(shops); shops = mergeShop(shops);
@@ -112,29 +124,48 @@ class CZDataBatch extends BaseBatch{
return rMap; return rMap;
} }
def districtRun(def districts){ def districtRun(def districts, def statMap, commids){
def total = districts.size(); def total = districts.size();
def index = 0; def totalCount = new AtomicLong()
ExecutorService executorService = Executors.newFixedThreadPool(20) final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<total;i++){ for(def i=0;i<total;i++){
final def district = districts[i] final def district = districts[i]
executorService.submit(new Runnable() { fixedThreadPool.execute(new Runnable() {
@Override
void run() { void run() {
index ++; try{
println index+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]"; totalCount.addAndGet(1)
converDistrict(district); converDistrict(district, statMap, commids);
println totalCount.get()+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
} }
}); });
} }
executorService.shutdown() latch.await()
executorService.awaitTermination(30, TimeUnit.MINUTES)
// def index = 0;
// ExecutorService executorService = Executors.newFixedThreadPool(20)
// for(def i=0;i<total;i++){
//
// final def district = districts[i]
// executorService.submit(new Runnable() {
// @Override
// void run() {
// index ++;
// println index+"/"+total +" ["+district.city+"-"+district.region+"-"+district.name+"]";
// converDistrict(district);
// }
// });
// }
// executorService.shutdown()
// executorService.awaitTermination(30, TimeUnit.MINUTES)
} }
def converDistrict(def dist){ def converDistrict(def dist, def statMap, def commids){
def map = [:] def map = [:]
//楼盘编号 //楼盘编号
/* def c_id = czIdMap.get(dist.city+"#"+dist.region+"#"+dist.name) /* def c_id = czIdMap.get(dist.city+"#"+dist.region+"#"+dist.name)
@@ -155,7 +186,11 @@ class CZDataBatch extends BaseBatch{
map.estate_type2 = dist.estate_type2 map.estate_type2 = dist.estate_type2
map.build_structure = dist.build_structure map.build_structure = dist.build_structure
// 楼盘均价 // 楼盘均价
map.avg_price = getDistrictPrice(dist.city, dist.region, dist.name, dist.address, dist.block); def priceObj = statMap.get(dist.region+"#"+dist.name)
if(priceObj && priceObj.avg_price){
map.avg_price = priceObj.avg_price
}
//map.avg_price = getDistrictPrice(dist.city, dist.region, dist.name, dist.address, dist.block);
// 省 市 区 // 省 市 区
if(dist.city == "常州"){ if(dist.city == "常州"){
map.province_code = regionMap.get("江苏"); map.province_code = regionMap.get("江苏");
@@ -166,23 +201,25 @@ class CZDataBatch extends BaseBatch{
map.city = dist.city; map.city = dist.city;
map.region = dist.region; map.region = dist.region;
if(!commids.contains(map.num_id)){
// 公共服务配套
map.public_service_set_code = getPublicServiceSetCode(dist.city, dist.lat2, dist.lng2);
// 基础设施
map.facilities_code = "01";
// 居住社区成熟度
map.sophistication_code = getSophisticationCode(dist.city, dist.lat2, dist.lng2);
// 交通条件
map.traffic_code = getTrafficCode(dist.city, dist.lat2, dist.lng2);
// 泊车条件 /
map.stop_car_code = getStopCarCode(dist);
// /学区房类型
map.school_code = getSchoolCode(dist.city, dist.region, dist.lat2, dist.lng2) ;
}
// 地址 // 地址
map.address = dist.address map.address = dist.address
// 所属区域 // 所属区域
map.in_region_code = getRegionCode(dist.region); map.in_region_code = getRegionCode(dist.region);
// 公共服务配套
map.public_service_set_code = getPublicServiceSetCode(dist.city, dist.lat2, dist.lng2);
// 基础设施
map.facilities_code = "01";
// 居住社区成熟度
map.sophistication_code = getSophisticationCode(dist.city, dist.lat2, dist.lng2);
// 交通条件
map.traffic_code = getTrafficCode(dist.city, dist.lat2, dist.lng2);
// 泊车条件 /
map.stop_car_code = getStopCarCode(dist);
// /学区房类型
map.school_code = getSchoolCode(dist.city, dist.region, dist.lat2, dist.lng2) ;
//绿化率 //绿化率
map.virescence_percent = getVirescencePercent(dist.virescence_percent, map.build_date); map.virescence_percent = getVirescencePercent(dist.virescence_percent, map.build_date);
//容积率 //容积率
@@ -921,20 +958,20 @@ class CZDataBatch extends BaseBatch{
if(price){ if(price){
return price.avg_price; return price.avg_price;
}else{ }else{
//样例估价 // //样例估价
def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false); // def s = OpenApiClient.evaluateDistrictPriceBySimilarHouse3(city, region, name, 100, false);
try{ // try{
JSONObject jsonObject = JSONObject.fromObject(s); // JSONObject jsonObject = JSONObject.fromObject(s);
def map = DataHandleUtils.JsonToMap(jsonObject); // def map = DataHandleUtils.JsonToMap(jsonObject);
if(map.code==200){ // if(map.code==200){
if(map.result){ // if(map.result){
return map.result.avgPrice; // return map.result.avgPrice;
} // }
//
} // }
}catch(Exception e){ // }catch(Exception e){
//
} // }
//分期找总期 //分期找总期
@@ -1078,4 +1115,18 @@ class CZDataBatch extends BaseBatch{
return list; return list;
} }
def findAllJnBankDatas(city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("num_id", 1)
def dbcursor = DIST_COLL.find(q, keys)
def set = new HashSet()
for(def d : dbcursor){
set << d.num_id
}
return set;
}
} }
@@ -1,5 +1,6 @@
package com.fangjia.script.loudong package com.fangjia.script.loudong
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
@@ -16,8 +17,15 @@ class CqfdcChromeCrawl {
static main(args) { static main(args) {
def task = new CqfdcChromeCrawl(); // def task = new CqfdcChromeCrawl();
task.run() // task.run()
def name = "E3뵀짜" //69눙(방뺍C-7뵀짜) 방뺍5-5뵀짜
def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|눙|떠|뵀]짜?.*?")
println s
def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+데禱")
println unitNumStr
} }
def run(){ def run(){
@@ -1,10 +1,14 @@
package com.fangjia.script.loudong package com.fangjia.script.loudong
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
@@ -14,6 +18,8 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
class CqfdcMatch { class CqfdcMatch {
@@ -23,6 +29,10 @@ class CqfdcMatch {
static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018)
static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl)
public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) { static main(args) {
@@ -32,7 +42,41 @@ class CqfdcMatch {
// task.export2() // task.export2()
// task.matchLoudong() // task.matchLoudong()
task.addLoudong() // task.addLoudong()
// task.addFdcLoudong()
task.updateHouseLoudong()
}
def updateHouseLoudong(){
def q = new BasicDBObject()
q.put("city", "重庆")
def keys = new BasicDBObject()
keys.put("room_num", 1)
keys.put("house_floor", 1)
def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000)
def index = 0
for(def d : dbcursor){
index++
def room_num = d.room_num
def house_floor = d.house_floor
try{
if(room_num && house_floor && room_num.isNumber()){
if(house_floor.toInteger()>0){
if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){
def newValue = house_floor +"-"+room_num
println "[${index}] "+room_num +" "+house_floor+" "+newValue
def udata = new BasicDBObject()
udata.put("room_num",newValue)
HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false);
}
}
}
}catch(Exception e){
println d._id.toString()
}
}
} }
def export(){ def export(){
def q = new BasicDBObject() def q = new BasicDBObject()
@@ -157,29 +201,378 @@ class CqfdcMatch {
return rets; return rets;
} }
def addHouseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def keys = []
def index = 0
for(def d : dbcursor) {
index++
println index
if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){
continue
}
if(d.area.toDouble()<=25){
continue
}
def newRegion = RegionData.formatRegion(d.city, d.region)
if (newRegion) {
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if (houseNumStr) {
house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "")
}
if (house_num) {
def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
if (unitNum) {
unitNum = unitNum.replace("单元", "")
}
def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num
if(!keys.contains(key) && d.room_num) {
keys << key
def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num)
if(!count){
def dist = null
if(d.match_city){
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
}
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.district_name)
udata.put("house_num", house_num)
if(unitNum){
udata.put("house_num_unit", unitNum)
}
udata.put("room_num", d.room_num)
udata.put("house_area",d.area.toDouble())
udata.put("status", 0)
udata.put("from", "房谱网")
if(d.floor_heigth && d.floor_heigth.isNumber()){
udata.put("house_height", d.floor_heigth.toInteger())
}
if(d.floor && d.floor.isNumber()){
udata.put("house_floor", d.floor.toInteger())
}
if(dist){
udata.put("district_id", dist._id.toString())
udata.put("name_in_seaweed", dist.name)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
HOUSE_LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", district_name)
q.put("house_num", house_num)
if(unitNum){
q.put("unit_num", unitNum)
}
q.put("room_num", room_num)
def count = HOUSE_LOUDONG_LINE_COLL.count(q)
return count;
}
def findAllLoundongByHouseNum(){ def findAllLoundongByHouseNum(){
def q = new BasicDBObject() def q = new BasicDBObject()
def dbcursor = loudongConn.find(q).batchSize(5000) def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000)
def rets = [] def keys = []
def map = [:]
for(def d : dbcursor){ for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.district_name+"##"+d.house_num def newRegion = RegionData.formatRegion(d.city, d.region)
if(d.unit_num){ if(newRegion){
key = key +"##"+d.unit_num def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
} def house_num = null
def detailMap= map.get(key) if(houseNumStr){
if(!detailMap){ house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
detailMap = [:] }
map.put(key,detailMap) if(house_num){
} def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元")
if(unitNum){
unitNum = unitNum.replace("单元","")
}
def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num
if(unitNum){
key = key +"##"+unitNum
}
if(!keys.contains(key)){
keys << key
def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num)
println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}"
if(obj){
def udata = new BasicDBObject()
def areas = obj.areas
if(!areas.contains(d.area)){
areas << d.area
if(detailMap.height){ def forms = obj.from
if(!forms.contains("房谱网")){
forms << "房谱网"
}
obj.m_date = new Date()
def query = new BasicDBObject("_id", obj._id)
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
}
}else{
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.district_name)
udata.put("house_num", house_num)
if(unitNum){
udata.put("unit_num", unitNum)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("areas",[d.area])
udata.put("from",["房谱网"])
if(d.floor_heigth && d.floor_heigth.toString().toInteger()){
udata.put("floors", d.floor_heigth.toString().toInteger())
}
udata.put("c_date", new Date())
udata.put("status", 1)
LOUDONG_LINE_COLL.save(udata)
}
}
}
} }
} }
return rets;
} }
def addFdcHouseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000)
def keys = []
def index = 0
for(def d : dbcursor){
index++
println index
if(!d.use_type){
continue
}
if(!d.use_type.contains("住宅")){
continue
}
def newRegion = d.match_region
if(newRegion){
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if(houseNumStr){
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
}
if(house_num){
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num
if(!keys.contains(key)) {
keys << key
def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num)
if(!count){
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
def dist = null
if(d.match_city){
dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district")
}
def floor = null
if(d.room_num && d.room_num.contains("-")) {
def arrs = d.room_num.split("-")
floor = arrs[0]
}
Integer room = null
Integer hall = null
if(d.room_type && d.room_type.contains("室")){
def roomStr = d.room_type.substring(0,d.room_type.indexOf("室"))
if(roomStr){
roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
room = roomStr.toInteger()
}
if(d.room_type.contains("厅")){
def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅"))
if(hallStr){
hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10")
hall = hallStr.toInteger()
}
}
}
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.name)
udata.put("house_num", house_num)
if(d.unit_num){
udata.put("house_num_unit", d.unit_num)
}
udata.put("room_num", d.room_num)
udata.put("house_area",d.area.toDouble())
udata.put("status", 0)
udata.put("from", "重庆网上房地产")
udata.put("house_height", maxHeight)
if(room){
udata.put("room", room)
}
if(hall){
udata.put("hall", hall)
}
if(floor && floor.isNumber()){
udata.put("house_floor", floor.toInteger())
}
if(dist){
udata.put("district_id", dist._id.toString())
udata.put("name_in_seaweed", dist.name)
}
if(d.address){
def address = AddressUtil.formatAddress(d.city, d.address)
if(address){
udata.put("address", address)
}
}
udata.put("c_date", new Date())
udata.put("m_date", new Date())
HOUSE_LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
public String getMatchString(String text,String pattern){
Pattern p=Pattern.compile(pattern);
Matcher m=p.matcher(text);
if(m.find()){
return m.group(0)
}
return null;
}
def addFdcBaseLoudong(){
def q = new BasicDBObject()
def dbcursor = loudongFdcConn.find(q).batchSize(5000)
def keys = []
for(def d : dbcursor){
if(!d.match_region){
continue
}
def newRegion = d.match_region
def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?")
def house_num = null
if(houseNumStr){
house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","")
}
if(house_num){
def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num
if(d.unit_num) {
key = key + "##" + d.unit_num
if (!keys.contains(key)) {
keys << key
def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num)
println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}"
def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num)
if (obj) {
def areas = obj.areas
if (!areas.contains(d.area)) {
areas << d.area
def forms = obj.from
if (!forms.contains("重庆网上房地产")) {
forms << "重庆网上房地产"
}
obj.m_date = new Date()
def query = new BasicDBObject("_id", obj._id)
LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj))
}
} else {
def udata = new BasicDBObject()
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", d.name)
udata.put("house_num", house_num)
if (d.unit_num) {
udata.put("unit_num", d.unit_num)
}
if (d.address) {
def address = AddressUtil.formatAddress(d.city, d.address)
if (address) {
udata.put("address", address)
}
}
udata.put("areas", [d.area])
udata.put("from", ["重庆网上房地产"])
udata.put("floors", maxHeight)
udata.put("c_date", new Date())
udata.put("status", 1)
LOUDONG_LINE_COLL.save(udata)
}
}
}
}
}
}
def findMaxHeight(city, name,house_num, unit_num){
def q = new BasicDBObject()
q.put("city",city)
q.put("name",name)
q.put("house_num_desc",house_num)
q.put("unit_num",unit_num)
def dbcursor = loudongFdcConn.find(q)
def heights = []
for(def h : dbcursor){
if(h.nominal_floor && h.nominal_floor.isNumber()){
def height = h.nominal_floor.toInteger()
if(!heights.contains(height)){
heights << height
}
}
}
if(heights){
heights = heights.sort()
return heights[heights.size()-1]
}
return null
}
def findLoudongOnLine(city, region, name, house_num, unit_num){
def q = new BasicDBObject()
q.put("city",city)
q.put("region",region)
q.put("name",name)
q.put("house_num",house_num)
if(unit_num){
q.put("unit_num",unit_num)
}
def dbo = LOUDONG_LINE_COLL.findOne(q)
return dbo
}
def stat(){ def stat(){
def q = new BasicDBObject() def q = new BasicDBObject()
@@ -291,7 +684,51 @@ class CqfdcMatch {
} }
def addLoudong(){ def addLoudong(){
// def stat = statBaseLoudong()
// println stat
// findAllLoundongByHouseNum();
def stat1 = statHouseLoudong()
println stat1
// addHouseLoudong()
}
def addFdcLoudong(){
// addFdcBaseLoudong()
addFdcHouseLoudong()
}
def statHouseLoudong(){
def q = new BasicDBObject()
q.put("city", "重庆")
def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q)
def total = dbcursor.size()
def ids = []
for(def d : dbcursor){
if(d.district_id && !ids.contains(d.district_id)){
ids << d.district_id
}
}
def map = [:]
map.rows_total = total
map.dist_count = ids.size()
return map
}
def statBaseLoudong(){
def q = new BasicDBObject()
q.put("city", "重庆")
def dbcursor = LOUDONG_LINE_COLL.find(q)
def total = dbcursor.size()
def ids = []
for(def d : dbcursor){
if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){
ids << d.id_in_seaweed
}
}
def map = [:]
map.rows_total = total
map.dist_count = ids.size()
return map
} }
@@ -0,0 +1,28 @@
package com.fangjia.script.temp.shenjian.didata
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
/**
* 从excel中匹配出
*/
ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row ->
def city = row['A']
def region = row['B']
def district_name = row['C']
def gaodeRegion = row['H']
def gaodeName = row['I']
def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict")
if (jsonObject != null && jsonObject.precise != -1) {
row['D'] = jsonObject.get("name")
}
if (gaodeRegion && gaodeName) {
def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict")
if (jsonObject2 != null && jsonObject2.precise != -1) {
row['J'] = jsonObject2.get("name")
}
}
});
@@ -0,0 +1,46 @@
package com.fangjia.script.temp.shenjian.geo
import com.fangjia.script.gj.location.Point
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xh.InsidePolygon
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017)
collection.find(new BasicDBObject()).forEach({ obj ->
if (obj.type == 'block') {
if (obj.region instanceof String) {
return;
}
BasicDBList reigon = (BasicDBList)obj.region
int count = 0
List<String> info = new ArrayList<>()
for (String s : reigon) {
info.add(s)
}
if (info.size() > 1) {
boolean find = false
for (String s : info) {
if (find) {
break
}
collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 ->
String border = obj2.border
String points = border.replaceAll(",", "-").replaceAll(";", "-")
List<com.fangjia.script.looplineinfo.Point> pointList = InsidePolygon.createPoint(points);
com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat)
if (InsidePolygon.aboutZone(point, pointList)) {
collection.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("region", obj2.name)));
find = true;
}
})
}
} else {
collection.update(new BasicDBObject("_id", obj._id),
new BasicDBObject('$set', new BasicDBObject("region", info.get(0))));
}
}
})
@@ -2,6 +2,8 @@ package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject import com.alibaba.fastjson.JSONObject
import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.ExcelUtils import com.fangjia.script.temp.shenjian.util.ExcelUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.temp.shenjian.util.HttpUtils
@@ -14,13 +16,15 @@ import java.util.concurrent.atomic.AtomicLong
class GetPoiFromExcel { class GetPoiFromExcel {
static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi"
static final ExecutorService executorService = Executors.newFixedThreadPool(10) static final ExecutorService executorService = Executors.newFixedThreadPool(10)
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static void main(String[] args) { static void main(String[] args) {
AtomicLong atomicLong = new AtomicLong(0) AtomicLong atomicLong = new AtomicLong(0)
//List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false); //List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false);
List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false); List<File> fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false);
for (File file : fileList) { for (File file : fileList) {
List<List<String>> seaweeds = ExcelUtils.read(file.getPath()) List<List<String>> seaweeds = ExcelUtils.read(file.getPath())
for (List<String> infoTemp : seaweeds) { for (List<String> infoTemp : seaweeds) {
@@ -31,9 +35,10 @@ class GetPoiFromExcel {
String city = info.get(0) String city = info.get(0)
String region = info.get(1) String region = info.get(1)
String seaweedName = info.get(2) String seaweedName = info.get(2)
List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302"); List<AmapPoi> pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes);
if (CollectionUtils.isNotEmpty(pois)) { if (CollectionUtils.isNotEmpty(pois)) {
String jsonStr = JSONObject.toJSONString(pois) List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr) HttpUtils.postForJson(URL, jsonStr)
atomicLong.incrementAndGet() atomicLong.incrementAndGet()
} }
@@ -13,7 +13,13 @@ import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import com.mongodb.client.model.Filters
import com.mongodb.client.model.UpdateOneModel
import com.mongodb.client.model.UpdateOptions
import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils import org.apache.commons.collections.CollectionUtils
import org.bson.Document
import org.bson.conversions.Bson
import org.bson.types.ObjectId import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
@@ -27,74 +33,85 @@ import java.util.concurrent.Executors
*/ */
class SeaweedAroundPoiSingal { class SeaweedAroundPoiSingal {
static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi"
static void main(String[] args) { static void main(String[] args) {
def types = listAllType() getNewSeaweed()
//def types = "0601" // def types = listAllType()
DBObject query = new BasicDBObject() // //def types = "0601"
query.put("status", 1) // DBObject query = new BasicDBObject()
conn136.find(query).batchSize(30).each {objTemp -> // query.put("status", 1)
final def obj = objTemp // conn136.find(query).batchSize(30).each {objTemp ->
def city = obj.city // final def obj = objTemp
def lng = obj.lng // def city = obj.city
def lat = obj.lat // def lng = obj.lng
List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") // def lat = obj.lat
if (CollectionUtils.isNotEmpty(pois)) { // List<AmapPoi> pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "")
String jsonStr = JSONObject.toJSONString(pois) // if (CollectionUtils.isNotEmpty(pois)) {
HttpUtils.postForJson(URL, jsonStr) // String jsonStr = JSONObject.toJSONString(pois)
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) // HttpUtils.postForJson(URL, jsonStr)
} else { // conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI")))
conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) // } else {
} // conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空")))
println("已处理小区:" + obj) // }
} // println("已处理小区:" + obj)
// }
} }
static getNewSeaweed() { static getNewSeaweed() {
DBObject query = new BasicDBObject() DBObject query = new BasicDBObject()
query.put("cat", "district") query.put("cat", "district")
query.put("visible", 0) query.put("visible", 0)
//query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a"))
DBObject filter = new BasicDBObject() DBObject filter = new BasicDBObject()
filter.put("_id", 1) filter.put("_id", 1)
filter.put("city", 1) filter.put("city", 1)
filter.put("region", 1) filter.put("region", 1)
filter.put("name", 1) filter.put("name", 1)
filter.put("alias", 1)
filter.put("address", 1)
filter.put("addr_alias", 1)
filter.put("lng", 1) filter.put("lng", 1)
filter.put("lat", 1) filter.put("lat", 1)
List list = [] List<WriteModel<Document>> writeModelList = new ArrayList<>()
def count = 0
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { Document updateBson = new Document()
// 已存在小区,无需同步 Document update = new Document()
} else { update.put("city", obj.city)
DBObject newObj = new BasicDBObject() update.put("region", obj.region)
newObj.put("_id", obj._id) update.put("name", obj.name)
newObj.put("city", obj.city) update.put("alias", obj.alias)
newObj.put("region", obj.region) update.put("address", obj.address)
newObj.put("name", obj.name) update.put("addr_alias", obj.addr_alias)
try { try {
newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) { } catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj) print("经纬度为空:" + obj._id + " " + obj)
ex.printStackTrace() }
} update.put("status", 1)
newObj.put("status", 1) update.put("flag", "未精确获取高德信息")
newObj.put("flag", "未获取周边POI") updateBson.append('$set', update)
list << newObj Bson filterBson = Filters.eq("_id", obj._id)
if (list.size() == 5000) { UpdateOptions updateOptions = new UpdateOptions();
conn136.insert(list) updateOptions.upsert(true)
list.clear() UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
} writeModelList.add(updateOneModel)
if (writeModelList.size() == 5000) {
conn136.bulkWrite(writeModelList)
count += 5000
println(count)
writeModelList.clear()
} }
} }
conn136.insert(list) if (writeModelList.size() > 0) {
list.clear() conn136.bulkWrite(writeModelList)
writeModelList.clear()
}
} }
static def getByIdAndInsert(String id) { static def getByIdAndInsert(String id) {
@@ -0,0 +1,163 @@
package com.fangjia.script.temp.shenjian.poi
import com.alibaba.fastjson.JSONObject
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.temp.shenjian.util.CommonDtoUtils
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.wenyan.auto.KeywordSearchUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.model.Filters
import com.mongodb.client.model.UpdateOneModel
import com.mongodb.client.model.UpdateOptions
import com.mongodb.client.model.WriteModel
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.bson.conversions.Bson
import org.bson.types.ObjectId
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* 小区POI信息精确查找
*
* @author shenjian
* @since 2022/5/25
*/
class SeaweedPoiDetail {
static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017")
static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail"
// 住宅类
static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304";
static final ExecutorService executorService = Executors.newFixedThreadPool(20)
static void main(String[] args) {
DBObject query = new BasicDBObject()
query.put("status", 1)
conn136.find(query).batchSize(1000).each {objTemp ->
final def obj = objTemp
executorService.submit(new Runnable() {
@Override
void run() {
def city = obj.city
def region = obj.region
def name = obj.name
def alias = obj.alias
def address = obj.address
def addr_alias = obj.addr_alias
StringJoiner stringJoiner = new StringJoiner("|")
// 主小区名搜索
stringJoiner.add(name)
// 根据别名搜
if (alias != null && CollectionUtils.isNotEmpty(alias)) {
for (String str : alias) {
stringJoiner.add(str)
}
}
// 根据路名搜
if (StringUtils.isNotBlank(address)) {
stringJoiner.add(address)
}
// 根据路别名搜
if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) {
for (String str : addr_alias) {
stringJoiner.add(str)
}
}
String keywords = stringJoiner.toString()
List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes)
if (CollectionUtils.isEmpty(pois)) {
pois = GaoDeUtils.searchPoi(city, name, region, typecodes)
}
if (CollectionUtils.isEmpty(pois)) {
pois = GaoDeUtils.searchPoi(city, address, region, typecodes)
}
List<AmapPoiDetail> detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois);
// for (AmapPoi amapPoi : pois) {
// List<AmapPoiDetail> amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId());
// if (CollectionUtils.isNotEmpty(amapPoiList)) {
// detailList.addAll(amapPoiList)
// }
// }
if (CollectionUtils.isNotEmpty(detailList)) {
String jsonStr = JSONObject.toJSONString(detailList)
HttpUtils.postForJson(URL, jsonStr)
conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到")))
} else {
conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区")))
}
println("已处理小区:" + obj)
}
})
}
executorService.shutdown()
}
static getNewSeaweed() {
DBObject query = new BasicDBObject()
query.put("cat", "district")
query.put("visible", 0)
DBObject filter = new BasicDBObject()
filter.put("_id", 1)
filter.put("city", 1)
filter.put("region", 1)
filter.put("name", 1)
filter.put("alias", 1)
filter.put("address", 1)
filter.put("addr_alias", 1)
filter.put("lng", 1)
filter.put("lat", 1)
List<WriteModel<Document>> writeModelList = new ArrayList<>()
def count = 0
DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj ->
Document updateBson = new Document()
Document update = new Document()
update.put("city", obj.city)
update.put("region", obj.region)
update.put("name", obj.name)
update.put("alias", obj.alias)
update.put("address", obj.address)
update.put("addr_alias", obj.addr_alias)
try {
update.put("lng", SeaweedUtils.formatLatLng(obj.lng))
update.put("lat", SeaweedUtils.formatLatLng(obj.lat))
} catch (Exception ex) {
print("经纬度为空:" + obj._id + " " + obj)
}
update.put("status", 1)
update.put("flag", "未精确获取高德信息")
updateBson.append('$set', update)
Bson filterBson = Filters.eq("_id", obj._id)
UpdateOptions updateOptions = new UpdateOptions();
updateOptions.upsert(true)
UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions);
writeModelList.add(updateOneModel)
if (writeModelList.size() == 5000) {
conn136.bulkWrite(writeModelList)
count += 5000
println(count)
writeModelList.clear()
}
}
if (writeModelList.size() > 0) {
conn136.bulkWrite(writeModelList)
writeModelList.clear()
}
}
}
@@ -0,0 +1,40 @@
package com.fangjia.script.temp.shenjian.util;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
/**
* 通用实体与DTO互相转换工具类
*
* @author shenjian
* @date 2021/06/16
*/
public class CommonDtoUtils {
public static <T> T transformV(Class<T> targetClass, Object source) {
if (source == null) {
return null;
} else {
try {
String jsonSource = JSON.toJSONString(source);
return JSONObject.parseObject(jsonSource, targetClass);
} catch (Exception var3) {
throw var3;
}
}
}
public static <T> List<T> transformListV(Class<T> targetClass, List<?> listSource) {
if (listSource == null) {
return null;
} else {
try {
String jsonSource = JSON.toJSONString(listSource);
return JSONArray.parseArray(jsonSource, targetClass);
} catch (Exception var3) {
throw var3;
}
}
}
}
@@ -7,10 +7,12 @@ import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.HttpRequest
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.statistics.poi.AmapPoiDetail
import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.util.HttpRequestUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.openqa.selenium.json.Json
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
@@ -88,12 +90,25 @@ class GaoDeUtils {
if (json && json.info && json.info == "OK") { if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger() count = json.getString("count").toInteger()
result.addAll(DataUtil.toMongoDbObj(json.pois)) result.addAll(DataUtil.toMongoDbObj(json.pois))
} else if (json && json.infocode && json.infocode == 10003) { } else if (json && json.infocode && json.infocode == "10044") {
throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") synchronized (this) {
} else if (json && json.infocode && json.infocode == 10004) { for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
break;
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000) Thread.sleep(60000)
break; break;
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
System.exit(1)
} else { } else {
break break
} }
@@ -146,7 +161,7 @@ class GaoDeUtils {
synchronized (this) { synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) { for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) { if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) { if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1) System.exit(1)
} }
keyUsed = new AtomicInteger(i+1) keyUsed = new AtomicInteger(i+1)
@@ -160,7 +175,9 @@ class GaoDeUtils {
break; break;
} else if (json && json.infocode && json.infocode == "10017") { } else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用 // 所请求的资源不可用
System.exit(1) //System.exit(1)
Thread.sleep(60000)
break;
} else { } else {
break break
} }
@@ -222,12 +239,65 @@ class GaoDeUtils {
} catch(Exception ex) { } catch(Exception ex) {
ex.printStackTrace() ex.printStackTrace()
} }
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") { } else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000) Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") { } else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用 // 所请求的资源不可用
System.exit(1) //System.exit(1)
Thread.sleep(60000)
}
return Collections.emptyList()
}
/**
* 获取详情
*/
static List<AmapPoiDetail> getAmapDetailById(id){
def params = [:]
params.key = GAO_DE_AK[keyUsed.intValue()]
params.id = id
params.output = 'json'
JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
if (json == null) {
json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params);
}
if (json && json.info && json.info == "OK") {
try {
List<AmapPoiDetail> ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class)
return ret
} catch(Exception ex) {
ex.printStackTrace()
}
} else if (json && json.infocode && json.infocode == "10044") {
synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000)
} else if (json && json.infocode && json.infocode == "10017") {
// 所请求的资源不可用
//System.exit(1)
Thread.sleep(60000)
} }
return Collections.emptyList() return Collections.emptyList()
} }
@@ -270,7 +340,7 @@ class GaoDeUtils {
synchronized (this) { synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) { for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) { if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) { if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1) System.exit(1)
} }
keyUsed = new AtomicInteger(i+1) keyUsed = new AtomicInteger(i+1)
@@ -278,13 +348,13 @@ class GaoDeUtils {
} }
break; break;
} }
} else if (json && json.infocode && json.infocode == "10004") { } else if (json && json.infocode && (json.infocode == "10004" || json.infocode == "10021" || json.infocode == "10017")) {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000) Thread.sleep(60000)
break; break;
} else if (json && json.infocode && json.infocode == "10017") { } else if (json && json.infocode && (json.infocode == "20000")) {
// 所请求的资源不可用 // 无效参数
System.exit(1) break;
} }
} }
return result return result
@@ -326,7 +396,7 @@ class GaoDeUtils {
synchronized (this) { synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) { for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) { if (GAO_DE_AK[i] == params.key) {
if (i + 1 > GAO_DE_AK.size()) { if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1) System.exit(1)
} }
keyUsed = new AtomicInteger(i+1) keyUsed = new AtomicInteger(i+1)
@@ -367,7 +437,16 @@ class GaoDeUtils {
return ["lng": arrs[0], "lat": arrs[1]] return ["lng": arrs[0], "lat": arrs[1]]
} }
} else if (json && json.infocode && json.infocode == "10044") { } else if (json && json.infocode && json.infocode == "10044") {
System.exit(1) synchronized (this) {
for (int i = 0; i < GAO_DE_AK.size(); i++) {
if (GAO_DE_AK[i] == params.key) {
if (i + 1 >= GAO_DE_AK.size()) {
System.exit(1)
}
keyUsed = new AtomicInteger(i+1)
}
}
}
} else if (json && json.infocode && json.infocode == "10004") { } else if (json && json.infocode && json.infocode == "10004") {
// 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。
Thread.sleep(60000) Thread.sleep(60000)
@@ -128,10 +128,10 @@ class SeaweedUtils {
*/ */
static formatHouseholdCount(def household_count) { static formatHouseholdCount(def household_count) {
if (household_count && FieldValidatorUtils.checkValue(household_count) ) { if (household_count && FieldValidatorUtils.checkValue(household_count) ) {
if (Long.valueOf(household_count) < 50) { if (Integer.valueOf(household_count) < 50) {
return "" // »§Êý return "" // »§Êý
} }
return household_count return household_count.toInteger()
} }
} }
@@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{
def filterSpacialChar(def s){ def filterSpacialChar(def s){
def pattern = ~/[()!;\·]\[【】()]/ def pattern = ~/[()!;\·]\[【】()]/
def matcher = pattern.matcher(s) def matcher = pattern.matcher(s)
def count = matcher.getCount() def count = matcher.getCount()
if(count && count>0){ if(count && count>0){
return matcher.replaceAll('_') return matcher.replaceAll('_')
@@ -43,8 +43,8 @@ class DataStat {
def source = "58ͬ³Ç"// def source = "58ͬ³Ç"//
def citys = findCrawlCity(source, 202202); def citys = findCrawlCity(source, 202202);
for(def city : citys){ for(def city : citys){
def count1 = findCount(source, city, 202202); def count1 = findCount(source, city, 202203);
def count2 = findCount(source, city, 202203); def count2 = findCount(source, city, 202204);
println city +" "+count1+" "+count2 println city +" "+count1+" "+count2
// println count1 // println count1
// println "--- ----" // println "--- ----"
@@ -40,13 +40,13 @@ class Esf58Crawl extends BaseBatch{
def static useDistKeys = [] def static useDistKeys = []
static def totalCount = new AtomicLong() static def totalCount = new AtomicLong()
def cityList = ["成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413",
"괏땍":"bd|424","괏癸":"baoting|10367","켓퀼":"nn|845", "爛鹵":"yz|637","�庫":"st|783","揭鹵":"taizhou|693","뺨갛":"ha|968","纜쉭":"zhanjiang|791","쏩鹵":"jinzhou|2354","웹츠":"km|541","진鹵":"liuzhou|7133","凉헤":"zq|901","젯暾멍":"lyg|2049","휑亮":"rizhao|3177","끽돠":"changde|872", "괏땍":"bd|424","괏癸":"baoting|10367","켓퀼":"nn|845", "爛鹵":"yz|637","�庫":"st|783","揭鹵":"taizhou|693","뺨갛":"ha|968","纜쉭":"zhanjiang|791","쏩鹵":"jinzhou|2354","웹츠":"km|541","진鹵":"liuzhou|7133","凉헤":"zq|901","젯暾멍":"lyg|2049","휑亮":"rizhao|3177","끽돠":"changde|872",
"꾕鹵":"cangzhou|652", "홋鹵":"qz|291","헌陶":"qingyuan|7303", "鮫퀼":"xn|2052","潾屢":"zunyi|7620","瓏碌":"zhuzhou|1086","뱍捺":"gy|2015","纛捺":"yy|821","其鹵":"xz|471", "벗랑깝":"hrb|202","露끗":"yc|858", "읜렌":"lf|772","해뽈돎":"qhd|1078","쁨鹵":"huizhou|722","쭤捺":"luoyang|556", "꾕鹵":"cangzhou|652", "홋鹵":"qz|291","헌陶":"qingyuan|7303", "鮫퀼":"xn|2052","潾屢":"zunyi|7620","瓏碌":"zhuzhou|1086","뱍捺":"gy|2015","纛捺":"yy|821","其鹵":"xz|471", "벗랑깝":"hrb|202","露끗":"yc|858", "읜렌":"lf|772","해뽈돎":"qhd|1078","쁨鹵":"huizhou|722","쭤捺":"luoyang|556",
"柯소鏤":"sjz|241","格覩":"ty|740","吉냘":"yancheng|613","斤捺":"xy|8694","켓념":"nanchong|2378","켓捺":"ny|592","衢쳔":"xm|606", "柯소鏤":"sjz|241","格覩":"ty|740","吉냘":"yancheng|613","斤捺":"xy|8694","켓념":"nanchong|2378","켓捺":"ny|592","衢쳔":"xm|606",
"축捺":"mianyang|1057","셌퀼":"jining|450","벳덱":"hd|572", "축捺":"mianyang|1057","셌퀼":"jining|450","벳덱":"hd|572",
"뺨굇":"huaibei|9357","뇔鹵":"chuzhou|10266", "뺨굇":"huaibei|9357","뇔鹵":"chuzhou|10266",
"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", "蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158",
"읊鹵":"lz|952","북례":"hf|837", "魁쨀컁펭":"xj|984","켓끗":"nc|669","�捺":"sy|188","鮫갛":"xa|483","藿렌":"wf|362", "읊鹵":"lz|952","북례":"hf|837", "魁쨀컁펭":"xj|984","켓끗":"nc|669","�捺":"sy|188","鮫갛":"xa|483","藿렌":"wf|362",
"交빤":"wuhu|2045","땜賻":"dg|413","끽鹵":"cz|463","뤘�":"fs|222", "交빤":"wuhu|2045","땜賻":"dg|413","끽鹵":"cz|463","뤘�":"fs|222",
"켓繫":"nt|394","섈주":"jl|700","櫓�":"zs|771","막捺":"fy|2325", "켓繫":"nt|394","섈주":"jl|700","櫓�":"zs|771","막捺":"fy|2325",
@@ -192,7 +192,8 @@ class Esf58Crawl extends BaseBatch{
void process(Object args) { void process(Object args) {
//addContent("르鹵","https://fz.58.com/xiaoqu/", 70); //addContent("르鹵","https://fz.58.com/xiaoqu/", 70);
def citys = ["北海","抚顺","大庆","呼伦贝尔","晋城","渭南","濮阳","河源","齐齐哈尔","曲靖","舟山","运城","巴中","怀化","铁岭","佳木斯","朝阳","宁德","葫芦岛","通辽","益阳","资阳","荆门","鄂州","大理","辽阳","阳泉","榆林","梧州","乌兰察布","黔南","巴音郭楞","广元","昌吉","玉溪","揭阳","随州","汕尾","北京","玉林","盘锦","六盘水","抚州","景德镇","四平","恩施","安顺","伊犁","毕节","丽水","天水","通化","阜新","攀枝花","阿克苏"] // "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州",
def citys = ["唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","北京","衡阳","南京","滨州","张家口","商丘","赣州","湖州","安庆","聊城","枣庄","阜阳","西安","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳"]
for(def city : citys){ for(def city : citys){
def str = cityList.get(city) def str = cityList.get(city)
for(def i=0;i<1;i++){ for(def i=0;i<1;i++){
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
//"fang.com":"搜房", //"fang.com":"搜房",
def sourceMap = ["fang.com":"搜房","ke.com":"贝壳找房","5i5j.com":"我爱我家"] def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"]
static main(args) { static main(args) {
def task = new SynchDistrictPrice() def task = new SynchDistrictPrice()
task.run() task.run()
@@ -26,7 +26,7 @@ class SynchDistrictPrice {
def run(){ def run(){
def citys = CommonUtils.findAllCity() def citys = ["重庆"]//CommonUtils.findAllCity()
sourceMap.each { k, v -> sourceMap.each { k, v ->
for(def city : citys){ for(def city : citys){
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){ def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
for(def source :sources){ for(def source :sources){
def count1 = findTotalCount(source, 202202); def count1 = findTotalCount(source, 202203);
def count2 = findTotalCount(source, 202203); def count2 = findTotalCount(source, 202204);
println source +" "+count1 +" "+count2 println source +" "+count1 +" "+count2
} }
} }
@@ -169,7 +169,7 @@ public class CommonUtils {
static void sendMessage(String queueName, byte[] bytes){ static void sendMessage(String queueName, byte[] bytes){
ConnectionFactory factory = new ConnectionFactory(); ConnectionFactory factory = new ConnectionFactory();
factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f"); factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f");
Connection conn = factory.newConnection(); Connection conn = factory.newConnection();
Channel channel = conn.createChannel(); Channel channel = conn.createChannel();
channel.queueDeclare(queueName, true, false, false, null); channel.queueDeclare(queueName, true, false, false, null);
@@ -92,11 +92,6 @@ public class DbUtils{
} }
} }
public static void executeQueryOnSeawater(BasicDBObject query,Closure c, BasicDBObject fields = null){
seawater.find(query,fields).batchSize(10000).each(c)
}
public static void executeQueryOnSeaweed(BasicDBObject query,Closure c, BasicDBObject fields = null){ public static void executeQueryOnSeaweed(BasicDBObject query,Closure c, BasicDBObject fields = null){
SEAWEED_SLAVE.find(query,fields).batchSize(1000).each(c) SEAWEED_SLAVE.find(query,fields).batchSize(1000).each(c)
} }
@@ -171,10 +166,6 @@ public class DbUtils{
return SEAWEED_SLAVE.findOne(query,fields); return SEAWEED_SLAVE.findOne(query,fields);
} }
public static int executeCountOnSeawater(BasicDBObject query){
return seawater.count(query);
}
public static BasicDBObject buildSeaweedQuery(def city = null, def region = null, def name = null, def cat = "district", def visible = 0){ public static BasicDBObject buildSeaweedQuery(def city = null, def region = null, def name = null, def cat = "district", def visible = 0){
BasicDBObject query = new BasicDBObject() BasicDBObject query = new BasicDBObject()
if(city){ if(city){
@@ -387,31 +378,6 @@ public class DbUtils{
return district.lat && district.lng && district.lng > 0 && district.lat > 0 return district.lat && district.lng && district.lng > 0 && district.lat > 0
} }
/**
* 记录修改小区库的操作日志
* @param city
* @param region
* @param name
* @return
*/
def static saveDeleteLog(String city, String region,String name,String category,int status){
if(status!=1){
return
}
DBObject obj = new BasicDBObject()
obj.put("city", city);
obj.put("region", region);
obj.put("district_name", name);
obj.put("category" , category)
obj.put("origin" , "脚本")
if(status==1){
obj.put("type" , "remove")
}
obj.put("c_date" , new Date())
obj.put("m_date" , new Date())
OPERATION_LOG.save(obj)
}
public static DBCollection getCollection(String host, int port, String dbName ,String collName){ public static DBCollection getCollection(String host, int port, String dbName ,String collName){
def collection = COLL_CACHE["${host}#${port}#${dbName}#${collName}"] def collection = COLL_CACHE["${host}#${port}#${dbName}#${collName}"]
if(!collection){ if(!collection){
@@ -229,6 +229,153 @@ public class OpenApiClient {
return null return null
} }
public static JSONObject searchLoudong(String id, String houseNum, String unitNum){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("id", id);
params.put("houseNumber", houseNum);
params.put("unitNumber", unitNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8")
println("${id} ${houseNum} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${id} ${houseNum}")
}
}
} catch (Exception e) {
println("${id} ${houseNum} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("id", id);
params.put("houseNumber", houseNum);
params.put("unitNumber", unitNum);
params.put("roomNumber", roomNum);
params.put("token", TOKEN);
String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8")
println("${id} ${houseNum} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${id} ${houseNum}")
}
}
} catch (Exception e) {
println("${id} ${houseNum} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject webankEvaluate(String city, String region, String address, String area){
try{
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", region);
params.put("address", address);
params.put("area", area.toString());
params.put("token", TOKEN);
String content = new HttpRequest().get("http://open.fangjia.com/webank/evaluate", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/webank/evaluate", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if(result){
if (result.getInt("code") == 200) {
return result;
} else if (result.getInt("code") !=0) {
println("${city} ${address} 程序调用异常")
}
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standard2Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if (result.getInt("code") == 0) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standard1Action(String city, String address) {
try {
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("token", TOKEN);
params.put("__refresh_cache","1")
String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
while (result == null) {
result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8")
println("${city} ${address} 程序调用异常")
Thread.sleep(6000)
}
if (result.getInt("code") == 200) {
return result.getJSONObject("result");
} else if (result.getInt("code") !=0) {
println("${city} ${address}")
}
} catch (Exception e) {
println("${city} ${address} ${e.getMessage()}")
e.printStackTrace();
}
return null
}
public static JSONObject standardize(String city, String address) { public static JSONObject standardize(String city, String address) {
try { try {
Map<String, String> params = new HashMap<String, String>(); Map<String, String> params = new HashMap<String, String>();
@@ -593,6 +740,91 @@ public class OpenApiClient {
} }
static JSONObject facilitys(String city, String region, String name) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city)
params.put("region", region)
params.put("name", name)
try {
String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("city", city)
params.put("district", region)
params.put("name", name)
params.put("startDate", startDate)
params.put("limit", "100")
try {
String content = new HttpRequest().get("http://open.fangjia.com/auction/searchPaimais", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject searchDealPrice(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startDate", startDate)
params.put("limit", limit.toString())
try {
String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject houseList(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startTime", startDate)
params.put("pageSize", limit.toString())
try {
String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject evaluaionList(String id, String startDate, limit) {
Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN);
params.put("id", id)
params.put("startDate", startDate)
params.put("limit", limit.toString())
try {
String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8")
JSONObject result = JSONObject.fromObject(content);
return result
} catch (Exception e) {
}
return null
}
static JSONObject districtDetail(String city, String region, String name) { static JSONObject districtDetail(String city, String region, String name) {
Map<String, String> params = new HashMap<String, String>(); Map<String, String> params = new HashMap<String, String>();
params.put("token", TOKEN); params.put("token", TOKEN);
@@ -0,0 +1,71 @@
package com.fangjia.script.statistics.poi;
import java.util.List;
/**
* @author shenjian
* @since 2022/5/25
*/
@lombok.NoArgsConstructor
@lombok.Data
public class AmapPoiDetail {
@com.alibaba.fastjson.annotation.JSONField(name = "id")
private String id;
@com.alibaba.fastjson.annotation.JSONField(name = "parent")
private List<?> parent;
@com.alibaba.fastjson.annotation.JSONField(name = "childtype")
private List<?> childtype;
@com.alibaba.fastjson.annotation.JSONField(name = "name")
private String name;
@com.alibaba.fastjson.annotation.JSONField(name = "type")
private String type;
@com.alibaba.fastjson.annotation.JSONField(name = "typecode")
private String typecode;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_type")
private List<?> bizType;
@com.alibaba.fastjson.annotation.JSONField(name = "address")
private String address;
@com.alibaba.fastjson.annotation.JSONField(name = "location")
private String location;
@com.alibaba.fastjson.annotation.JSONField(name = "tel")
private String tel;
@com.alibaba.fastjson.annotation.JSONField(name = "pname")
private String pname;
@com.alibaba.fastjson.annotation.JSONField(name = "cityname")
private String cityname;
@com.alibaba.fastjson.annotation.JSONField(name = "adname")
private String adname;
@com.alibaba.fastjson.annotation.JSONField(name = "importance")
private List<?> importance;
@com.alibaba.fastjson.annotation.JSONField(name = "shopid")
private List<?> shopid;
@com.alibaba.fastjson.annotation.JSONField(name = "shopinfo")
private String shopinfo;
@com.alibaba.fastjson.annotation.JSONField(name = "poiweight")
private List<?> poiweight;
@com.alibaba.fastjson.annotation.JSONField(name = "distance")
private String distance;
@com.alibaba.fastjson.annotation.JSONField(name = "biz_ext")
private BizExtDTO bizExt;
@com.alibaba.fastjson.annotation.JSONField(name = "photos")
private List<PhotosDTO> photos;
@lombok.NoArgsConstructor
@lombok.Data
public static class BizExtDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "rating")
private List<?> rating;
@com.alibaba.fastjson.annotation.JSONField(name = "cost")
private String cost;
}
@lombok.NoArgsConstructor
@lombok.Data
public static class PhotosDTO {
@com.alibaba.fastjson.annotation.JSONField(name = "title")
private List<?> title;
@com.alibaba.fastjson.annotation.JSONField(name = "url")
private String url;
}
}
@@ -1,12 +1,12 @@
package com.fangjia.service.address; package com.fangjia.service.address;
import com.fangjia.core.RegionData; import com.fangjia.core.RegionData;
import com.fangjia.core.util.*; import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeoUtils;
import com.fangjia.dal.QueryParam; import com.fangjia.dal.QueryParam;
import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea; import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.AmapPoi;
import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong;
import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.mongo.model.bkoff.Loudong;
import com.fangjia.dal.seaweed.dao.SeaweedDao; import com.fangjia.dal.seaweed.dao.SeaweedDao;
@@ -16,17 +16,15 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors;
import com.fangjia.service.address.processor.AddressProcessors; import com.fangjia.service.address.processor.AddressProcessors;
import com.fangjia.service.api.AliyunService; import com.fangjia.service.api.AliyunService;
import com.fangjia.service.bo.AliyunBo; import com.fangjia.service.bo.AliyunBo;
import com.fangjia.service.bo.MatchAddressScore;
import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.mongo.addrepo.AddressRepoService; import com.fangjia.service.mongo.addrepo.AddressRepoService;
import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.HouseLoudongService;
import com.fangjia.service.mongo.bkoff.LoudongService; import com.fangjia.service.mongo.bkoff.LoudongService;
import com.google.common.base.Function; import com.google.common.base.Function;
import com.google.common.collect.MapMaker; import com.google.common.collect.MapMaker;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger; import org.apache.log4j.Logger;
import org.bson.types.ObjectId;
import java.util.*; import java.util.*;
import java.util.concurrent.ConcurrentMap; import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
@@ -759,6 +757,13 @@ public class AddressParserServiceImpl implements AddressParserService {
double simScore; double simScore;
boolean onSameRoad = false; boolean onSameRoad = false;
boolean sameDistrict = isSameDistrict(address, metaData); boolean sameDistrict = isSameDistrict(address, metaData);
if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){
return 0;
}
// //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区
// if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){
// return 0;
// }
Iterator<Map.Entry<String, MatchText>> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator(); Iterator<Map.Entry<String, MatchText>> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator();
Map.Entry<String, MatchText> addrEntry; Map.Entry<String, MatchText> addrEntry;
while(addrEntrySet.hasNext()){ while(addrEntrySet.hasNext()){
@@ -808,7 +813,6 @@ public class AddressParserServiceImpl implements AddressParserService {
} }
} }
} }
String keywordPinYin = address.getKeywordsPinYin(); String keywordPinYin = address.getKeywordsPinYin();
String keywords = address.getKeywords(); String keywords = address.getKeywords();
String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin(); String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin();
@@ -892,7 +896,7 @@ public class AddressParserServiceImpl implements AddressParserService {
if(name.length() == 2 && (name.equals(address.getOrgAddress()) || name.equals(address.getEffectiveAddress()))){ if(name.length() == 2 && (name.equals(address.getOrgAddress()) || name.equals(address.getEffectiveAddress()))){
keywordScore = Math.max(keywordScore, namePriority); keywordScore = Math.max(keywordScore, namePriority);
}else if(name.length() > 3 && (address.getOrgAddrePinYin().contains(namePinYin) || enhanceContains(address.getOrgAddress(), name))){ }else if(name.length() > 3 && (address.getOrgAddrePinYin().contains(namePinYin) || enhanceContains(address.getOrgAddress(), name))){
double similar = AddressParserUtil.getSimilarityRatio(name, address.getEffectiveAddress()); double similar = AddressParserUtil.getSimilarityRatio(name, address.getOrgAddress());
if(similar == 1){ if(similar == 1){
keywordScore = Math.max(similar, keywordScore) * namePriority * 1.0001; keywordScore = Math.max(similar, keywordScore) * namePriority * 1.0001;
}else if(similar > 0.1){ }else if(similar > 0.1){
@@ -907,20 +911,27 @@ public class AddressParserServiceImpl implements AddressParserService {
if (standardAddressScore > 0 && keywordScore > 0) { if (standardAddressScore > 0 && keywordScore > 0) {
matchLevelScore = 0.9; matchLevelScore = 0.9;
similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE; similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE;
} else if(keywordScore > 0 && streetScore > 0){
matchLevelScore = 0.8;
similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE;
} else if (standardAddressScore > 0) { } else if (standardAddressScore > 0) {
matchLevelScore = 0.7; matchLevelScore = 0.8;
similarityScore = standardAddressScore * SIMILARITY_RATE; similarityScore = standardAddressScore * SIMILARITY_RATE;
} else if(keywordScore > 0 && streetScore > 0){
matchLevelScore = 0.7;
similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE;
} else if (keywordScore > 0) { } else if (keywordScore > 0) {
matchLevelScore = 0.6; matchLevelScore = 0.6;
similarityScore = keywordScore * SIMILARITY_RATE; similarityScore = keywordScore * SIMILARITY_RATE;
} else if (fuzzyKeywordsScore > 0) { } else if (fuzzyKeywordsScore > 0) {
matchLevelScore = 0.5; if(address.hasCoordinate() && metaData.getLng() != null && GeoUtils.calculateDistance(address.getLng(), address.getLat(),
metaData.getLng(), metaData.getLat()) < 100){
matchLevelScore = 0.7;
}else{
matchLevelScore = 0.4;
}
similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE; similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE;
}else if(metaData.getId().equals(address.getPolygonSeaweedId())){
matchLevelScore = 0.4;
similarityScore = 0.05;
} }
if (similarityScore > 0) { if (similarityScore > 0) {
if(metaData.getBuildingType() != null){ if(metaData.getBuildingType() != null){
if ("shop".equals(metaData.getBuildingType().toString())) { if ("shop".equals(metaData.getBuildingType().toString())) {
@@ -942,10 +953,6 @@ public class AddressParserServiceImpl implements AddressParserService {
* similarityScore = ÏàËÆÈ²¿·Ö(0.07) + ÀàÐͲ¿·Ö(0.01) < 0.1 * similarityScore = ÏàËÆÈ²¿·Ö(0.07) + ÀàÐͲ¿·Ö(0.01) < 0.1
* */ * */
double score = matchLevelScore + similarityScore; double score = matchLevelScore + similarityScore;
if(address.hasCoordinate() && metaData.hasCoordinate()
&& GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) > 3 * 1000){
score = 0d;
}
return score; return score;
} }
@@ -980,7 +987,7 @@ public class AddressParserServiceImpl implements AddressParserService {
return false; return false;
} }
if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){ if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){
return false; return true;
} }
String district = metaData.getDistrict(); String district = metaData.getDistrict();
return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict()); return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict());
@@ -61,6 +61,8 @@ public class Address {
private String suggestDistrict; private String suggestDistrict;
private String suggestName; private String suggestName;
private String suggestNamePinYin; private String suggestNamePinYin;
private String polygonSeaweedId;
private String geoRegion;
private List<String> buildingNumbers; private List<String> buildingNumbers;
private PreciseLevel preciseLevel; private PreciseLevel preciseLevel;
private boolean pinYin; private boolean pinYin;
@@ -750,6 +752,14 @@ public class Address {
return false; return false;
} }
public String getGeoRegion() {
return geoRegion;
}
public void setGeoRegion(String geoRegion) {
this.geoRegion = geoRegion;
}
@Override @Override
public String toString() { public String toString() {
return "Address{" + return "Address{" +
@@ -809,6 +819,14 @@ public class Address {
'}'; '}';
} }
public String getPolygonSeaweedId() {
return polygonSeaweedId;
}
public void setPolygonSeaweedId(String polygonSeaweedId) {
this.polygonSeaweedId = polygonSeaweedId;
}
public String toMd5Hex(){ public String toMd5Hex(){
String keyProperties = "Address{" + String keyProperties = "Address{" +
"city='" + city + '\'' + "city='" + city + '\'' +
@@ -20,18 +20,16 @@ public class AddressProcessors{
static { static {
//ÐÞ¸Ä˳ÐòÐèÒªµ÷Õûprocess i µÄÖµ //ÐÞ¸Ä˳ÐòÐèÒªµ÷Õûprocess i µÄÖµ
addressProcessors.add(new TokenProcessor());
addressProcessors.add(new ProximalCharactersProcessor()); addressProcessors.add(new ProximalCharactersProcessor());
addressProcessors.add(new PreNoiseProcessor()); addressProcessors.add(new PreNoiseProcessor());
addressProcessors.add(new AddressFormatProcessor()); addressProcessors.add(new AddressFormatProcessor());
addressProcessors.add(new TokenProcessor());
addressProcessors.add(new FloorProcessor()); addressProcessors.add(new FloorProcessor());
addressProcessors.add(new TooltipProcessor()); addressProcessors.add(new TooltipProcessor());
addressProcessors.add(new ProvinceProcessor()); addressProcessors.add(new ProvinceProcessor());
addressProcessors.add(new CityProcessor()); addressProcessors.add(new CityProcessor());
addressProcessors.add(new DistrictProcessor()); addressProcessors.add(new DistrictProcessor());
addressProcessors.add(new LoopLineProcessor());
addressProcessors.add(new TownshipProcessor()); addressProcessors.add(new TownshipProcessor());
addressProcessors.add(new LoopLineProcessor());
addressProcessors.add(new StageAndPartitionProcessor()); addressProcessors.add(new StageAndPartitionProcessor());
addressProcessors.add(new DirectionProcessor()); addressProcessors.add(new DirectionProcessor());
addressProcessors.add(new OffsetProcessor()); addressProcessors.add(new OffsetProcessor());
@@ -44,7 +42,7 @@ public class AddressProcessors{
addressProcessors.add(new NumericProcessor()); addressProcessors.add(new NumericProcessor());
addressProcessors.add(new ProvinceAdjustProcessor()); addressProcessors.add(new ProvinceAdjustProcessor());
addressProcessors.add(new FormatProcessor()); addressProcessors.add(new FormatProcessor());
// addressProcessors.add(new GeoProcessor()); addressProcessors.add(new GeoProcessor());
} }
@@ -52,8 +52,9 @@ public class CityProcessor extends AbstractAddressProcessor {
} }
if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){ if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){
String formatCity = RegionData.formatCity(city); String formatCity = RegionData.formatCity(city);
String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null; address.setCity(formatCity);
String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity; String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null;
String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity;
/**如果省份和城市信息相互矛盾 则用区域推断城市*/ /**如果省份和城市信息相互矛盾 则用区域推断城市*/
if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){ if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){
List<String> possibleCities = speculateCityNameByRegion(address, city); List<String> possibleCities = speculateCityNameByRegion(address, city);
@@ -191,15 +192,12 @@ public class CityProcessor extends AbstractAddressProcessor {
}else{ }else{
weight.score += 2; weight.score += 2;
} }
}else if(address.contains(city)){ }else if(address.contains(city)){
index = address.indexOf(city); index = address.indexOf(city);
weight.index = index; weight.index = index;
index += city.length(); index += city.length();
if(province != null && (orgAddress.contains(province + city) || orgAddress.contains(ProvinceData.getDisplay(province) + city))){ if(province != null && (orgAddress.contains(province + city) || orgAddress.contains(ProvinceData.getDisplay(province) + city))){
weight.score += 2; weight.score += 2;
}else{
weight.score += 1;
} }
} }
Map<String, List<String>> regionMap = RegionData.getAllRegions(city); Map<String, List<String>> regionMap = RegionData.getAllRegions(city);
@@ -208,9 +206,11 @@ public class CityProcessor extends AbstractAddressProcessor {
if(matchResult !=null){ if(matchResult !=null){
address = address.replace(matchResult.matchContent, ""); address = address.replace(matchResult.matchContent, "");
} }
if(address.indexOf(region, index) > -1){ if(address.indexOf(region + "Çø", index) > -1 || address.indexOf(region + "ÏØ", index) > -1){
if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){ if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){
weight.score += 2; weight.score += 2;
}else if(city.equals(addr.getCity())){
weight.score += 10;
}else{ }else{
weight.score += 1; weight.score += 1;
} }
@@ -221,9 +221,13 @@ public class CityProcessor extends AbstractAddressProcessor {
List<String> alias = regionMap.get(region); List<String> alias = regionMap.get(region);
if(alias != null){ if(alias != null){
for(String alia : alias){ for(String alia : alias){
if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1 if(StringUtils.isNotBlank(alia) && !alia.equals(region) && alia.length() > 2 && address.indexOf(alia, index) > -1
&& (alia.endsWith("市")|(alia.endsWith("县")))){ && (alia.endsWith("市")|(alia.endsWith("县")))){
weight.score += 1; if(city.equals(addr.getCity())){
weight.score += 10;
}else{
weight.score += 1;
}
weight.index = address.indexOf(alia, index); weight.index = address.indexOf(alia, index);
break; break;
} }
@@ -237,16 +241,6 @@ public class CityProcessor extends AbstractAddressProcessor {
} }
List<FragmentWeight> weights = new ArrayList<FragmentWeight>(weightMap.values()); List<FragmentWeight> weights = new ArrayList<FragmentWeight>(weightMap.values());
if(weights.size() > 0){ if(weights.size() > 0){
/*String street = getStreet(address);
if(street != null){
Iterator<FragmentWeight> iterator = weights.iterator();
while(iterator.hasNext()){
FragmentWeight fragmentWeight = iterator.next();
if(match(street, ".*" + fragmentWeight.text + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)")){
iterator.remove();
}
}
}*/
Collections.sort(weights); Collections.sort(weights);
} }
if(weights.size() > 0){ if(weights.size() > 0){
@@ -167,6 +167,9 @@ public class DistrictProcessor extends AbstractAddressProcessor {
address.setDevelopmentZone(dz); address.setDevelopmentZone(dz);
} }
} }
if(address.getSuggestDistrict() != null && address.getDistrict() == null){
address.setDistrict(address.getSuggestDistrict());
}
address.setAddress(orgAddress); address.setAddress(orgAddress);
address.setEffectiveAddress(address.getAddress()); address.setEffectiveAddress(address.getAddress());
return RepeatStatus.NEXT; return RepeatStatus.NEXT;
@@ -5,8 +5,12 @@ import com.fangjia.core.util.HttpRequest;
import com.fangjia.service.address.mode.Address; import com.fangjia.service.address.mode.Address;
import net.sf.json.JSONArray; import net.sf.json.JSONArray;
import net.sf.json.JSONObject; import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger;
import java.util.*;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/** /**
* µØÖ·¾­Î³È´¦ÀíÆ÷ * µØÖ·¾­Î³È´¦ÀíÆ÷
@@ -16,78 +20,69 @@ import java.util.*;
public class GeoProcessor extends AbstractAddressProcessor { public class GeoProcessor extends AbstractAddressProcessor {
private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde"; private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde";
private static final Logger logger = Logger.getLogger(GeoProcessor.class);
List<String> POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); List<String> POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口", "门牌号");
List<String> DISTRICT_LEVEL = Arrays.asList("区县", "开发区", "乡镇", "村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口");
@Override @Override
public RepeatStatus process(Address address) { public RepeatStatus process(Address address) {
if (address.hasCoordinate()) { if (address.hasCoordinate()) {
return RepeatStatus.NEXT; return RepeatStatus.NEXT;
} }
List<String> queryTypes = new ArrayList<String>(); String addr = address.getEffectiveAddress();
List<String> addressList = new ArrayList<String>(); if(addr == null || addr.length() < 3){
if (address.isStandardAddress()) { addr = address.getOrgAddress();
addressList.add(address.getStandardAddress());
queryTypes.add("StandardAddress");
}
if (StringUtils.isNotBlank(address.getOrgKeywords())) {
addressList.add(address.getOrgKeywords());
queryTypes.add("Keywords");
} }
HttpRequest request = new HttpRequest(); HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>(); Map<String, String> params = new HashMap<String, String>();
params.put("address", StringUtils.join(addressList, "|")); params.put("address", addr);
params.put("key", KEY); params.put("key", KEY);
params.put("city", address.getCity()); params.put("city", address.getCity());
params.put("batch", "true");
params.put("output", "json"); params.put("output", "json");
try { try {
JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8"); JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8");
logger.info("高德API:" + response.toString());
if (response.getInt("status") == 1) { if (response.getInt("status") == 1) {
JSONArray geocodes = response.getJSONArray("geocodes"); JSONArray geocodes = response.getJSONArray("geocodes");
if(queryTypes.contains("StandardAddress")){ if(geocodes.size() > 0){
JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("StandardAddress")); JSONObject json = geocodes.getJSONObject(0);
String level = geocode.getString("level"); if(json.get("district") instanceof String){
if(DISTRICT_LEVEL.contains(level)){ String city = address.getCity();
if(StringUtils.isBlank(address.getDistrict())){ if(city == null){
address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); city = RegionData.formatCity(json.getString("city"));
} }
String region = RegionData.formatRegion(city, json.getString("district"));
address.setGeoRegion(region);
} }
if(geocode.containsKey("street") && geocode.containsKey("number")){ if(POI_LEVELS.contains(json.getString("level"))){
String geocodeAddress = geocode.getString("street") + geocode.getString("number"); //对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼
if(address.getStandardAddress().contains(geocodeAddress)){ if("门牌号".equals(json.getString("level"))){
String location = geocode.getString("location"); if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){
address.setLng(Double.parseDouble(location.split(",")[0])); return RepeatStatus.NEXT;
address.setLat(Double.parseDouble(location.split(",")[1])); }
}
}
}
if(queryTypes.contains("Keywords")){
JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("Keywords"));
String level = geocode.getString("level");
if(DISTRICT_LEVEL.contains(level)){
if(StringUtils.isBlank(address.getDistrict())){
address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district")));
}
}
if(POI_LEVELS.contains(level)){
String formattedAddress = geocode.getString("formatted_address");
formattedAddress = formattedAddress
.replaceFirst(geocode.getString("province"), "")
.replaceFirst(geocode.getString("city"), "")
.replaceFirst(geocode.getString("district"), "");
if(formattedAddress.equals(address.getOrgKeywords())){
String location = geocode.getString("location");
address.setLng(Double.parseDouble(location.split(",")[0]));
address.setLat(Double.parseDouble(location.split(",")[1]));
} }
String location = json.getString("location");
address.setLng(Double.valueOf(location.split(",")[0]));
address.setLat(Double.valueOf(location.split(",")[1]));
String tmp = "{ \"_source\": [\"seaweed_id\"], \"query\": {\"geo_shape\": {\"shape\": {\"shape\": {\"type\": \"Point\",\"coordinates\":[location]},\"relation\": \"CONTAINS\"}}}}";
tmp = tmp.replace("location", location);
Map<String, String> headers = new HashMap<String, String>();
headers.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz");
JSONObject resp = JSONObject.fromObject(request.postWithBody("http://192.168.0.206:9200/seaweed_polygon/_search", JSONObject.fromObject(tmp),headers,"UTF-8"));
if(resp.containsKey("hits") && resp.getJSONObject("hits").containsKey("hits")){
JSONArray hits = resp.getJSONObject("hits").getJSONArray("hits");
if(hits.size() > 0){
String seaweedId = hits.getJSONObject(0).getJSONObject("_source").getString("seaweed_id");
address.setPolygonSeaweedId(seaweedId);
}
}
} }
} }
} }
} catch (Exception e) { } catch (Exception e) {
logger.error(e);
e.printStackTrace();
} }
return RepeatStatus.NEXT; return RepeatStatus.NEXT;
} }
@@ -22,7 +22,7 @@ public class KeywordProcessor extends AbstractAddressProcessor {
static final List<Pattern> KEYWORDS_PATTERNS = Arrays.asList(Pattern.compile(".*[^一二三四五六七八九十\\d]{3,}号")); static final List<Pattern> KEYWORDS_PATTERNS = Arrays.asList(Pattern.compile(".*[^一二三四五六七八九十\\d]{3,}号"));
static final List<String> KEYWORDS = Arrays.asList("居委会", "辖区", "街道办", "街道", "办事处", "村委", "管理区"); static final List<String> KEYWORDS = Arrays.asList("居委会", "辖区", "街道办", "街道", "办事处", "村委", "管理区");
static final List<Pattern> NOISE_PATTERNS = Arrays.asList(Pattern.compile("^.*\\d+队"), Pattern.compile("^\\d+号院$")); static final List<Pattern> NOISE_PATTERNS = Arrays.asList(Pattern.compile("^.*\\d+队"), Pattern.compile("^\\d+号院$"));
static final List<Pattern> INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属院")); static final List<Pattern> INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属[区院]"), Pattern.compile("宿舍"), Pattern.compile("生活区"), Pattern.compile("公寓"));
static final Pattern INVALID_PATTERN = Pattern.compile("^([-\\\\、])+$"); static final Pattern INVALID_PATTERN = Pattern.compile("^([-\\\\、])+$");
@@ -42,12 +42,10 @@ public class TooltipProcessor extends AbstractAddressProcessor {
if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){ if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){
address.setTooltip(tooltip); address.setTooltip(tooltip);
address.setStreet(getStreet(tooltip)); address.setStreet(getStreet(tooltip));
} else if(containsDistrictName(tooltip)){ }else if(RegionData.isRegion(address.getCity(), tooltip)){
address.setTooltip(tooltip);
} else if(RegionData.isRegion(address.getCity(), tooltip)){
address.setSuggestDistrict(tooltip); address.setSuggestDistrict(tooltip);
}else{ }else{
address.addNosie(tooltip); address.setTooltip(tooltip);
} }
} }
} }
@@ -15,15 +15,6 @@ public class VillageNameProcessor extends AbstractAddressProcessor {
@Override @Override
public AbstractAddressProcessor.RepeatStatus process(Address address) { public AbstractAddressProcessor.RepeatStatus process(Address address) {
String orgAddress = address.getAddress(); String orgAddress = address.getAddress();
if(address.getTokens()!=null){
for(Token token:address.getTokens()){
if("district".equals(token.getKeywordTypes())){
address.setKeywords(token.getName());
address.setAddress(orgAddress.replace(token.getName(),""));
return RepeatStatus.NEXT;
}
}
}
List<String> names = tokenDistrictNames(orgAddress); List<String> names = tokenDistrictNames(orgAddress);
List<String> kws = new ArrayList<String>(); List<String> kws = new ArrayList<String>();
for(String name : names){ for(String name : names){
@@ -1,24 +1,16 @@
package com.fangjia.service.cnAnalyzer; package com.fangjia.service.cnAnalyzer;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fangjia.core.util.DataUtil; import com.fangjia.core.util.DataUtil;
import com.goojia.solr.ganalytics.IKSegmentation; import com.goojia.solr.ganalytics.IKSegmentation;
import com.goojia.solr.ganalytics.Lexeme; import com.goojia.solr.ganalytics.Lexeme;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.*;
/** /**
* 房价网分词器 * 房价网分词器
@@ -240,7 +240,7 @@ public class AddressRepoServiceImpl implements AddressRepoService {
} }
private boolean isAvailablePoi(AddressMatchMetaData data){ private boolean isAvailablePoi(AddressMatchMetaData data){
if(data.getType() != null && data.getType().contains("·")){ if(data.getType() != null && (data.getType().contains("·") || data.getType().contains("½ÖµÀ"))){
return false; return false;
} }
return true; return true;
@@ -0,0 +1,9 @@
package com.fangjia.service.open;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityBasePriceService {
CommunityBasePrice getById(String id);
}
@@ -0,0 +1,8 @@
package com.fangjia.service.open;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityPoiService {
CommunityPoi getById(String id);
}
@@ -0,0 +1,22 @@
package com.fangjia.service.open.impl;
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.service.open.CommunityBasePriceService;
import com.fangjia.service.open.CommunityPoiService;
public class CommunityBasePriceServiceImpl implements CommunityBasePriceService {
private CommunityBasePriceDao communityBasePriceDao;
@Override
public CommunityBasePrice getById(String id) {
return communityBasePriceDao.findById(id);
}
public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) {
this.communityBasePriceDao = communityBasePriceDao;
}
}
@@ -0,0 +1,19 @@
package com.fangjia.service.open.impl;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.service.open.CommunityPoiService;
public class CommunityPoiServiceImpl implements CommunityPoiService {
private CommunityPoiDao communityPoiDao;
@Override
public CommunityPoi getById(String id) {
return communityPoiDao.findById(id);
}
public void setCommunityPoiDao(CommunityPoiDao communityPoiDao) {
this.communityPoiDao = communityPoiDao;
}
}
@@ -20,7 +20,7 @@
.officeMap{width:100%;height:400px;} .officeMap{width:100%;height:400px;}
.main_content.partner{overflow:hidden;} .main_content.partner{overflow:hidden;}
.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:24px;margin-bottom:25px;cursor:pointer;} .main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:15px;margin-bottom:25px;cursor:pointer;}
.partner a.partner11{background-position:0 0;} .partner a.partner11{background-position:0 0;}
.partner a.partner12{background-position:-204px 0;} .partner a.partner12{background-position:-204px 0;}