This commit is contained in:
2022-06-13 18:35:47 +08:00
157 changed files with 14710 additions and 1046 deletions
@@ -0,0 +1,202 @@
package com.fangjia.core.util;
import org.apache.commons.lang.StringUtils;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class SeaweedUtils {
public static void main(String[] args){
System.out.println(formatVirescencePercent("1"));
}
/**
* 标准化输出绿化率
*/
public static String formatVirescencePercent(String virescence_percent) {
// 绿化率
if (StringUtils.isNotBlank(virescence_percent) && checkValue(virescence_percent)!=null) {
double val = Double.valueOf(virescence_percent);
if (val < 1) {
return String.valueOf(val * 100);
}
if (val == 1) {
return String.valueOf(val * 10);
}
if (val >= 70) {
return "65";
}
return String.valueOf(val);
}
return null;
}
/**
* 标准化容积率输出
*/
public static String formatCubagePercent(String cubage_percent) {
if (StringUtils.isNotBlank(cubage_percent) && checkValue(cubage_percent)!=null) {
double val = Double.valueOf(cubage_percent);
if (val >= 7 && val < 8) {
return "6.2";
}
if (val >= 8 && val < 9) {
return "6.3";
}
if (val >= 9 && val < 10) {
return "6.5";
}
if (val >= 10) {
return "6.7";
}
return String.valueOf(val);
}
return null;
}
/**
* 标准化车位输出
*/
public static String formatCarSiteCount(String car_site_count) {
if (StringUtils.isNotBlank(car_site_count) && checkValue(car_site_count)!=null) {
long val = Long.valueOf(car_site_count);
if (val > 2500 || val < 100) {
return "";
}
return String.valueOf(val);
}
return null;
}
/**
* 标准化户数输出
*/
public static String formatHouseholdCount(String household_count) {
if (StringUtils.isNotBlank(household_count) && checkValue(household_count)!=null ) {
if (Integer.valueOf(household_count) < 50) {
return null; // 户数
}
return household_count;
}
return null;
}
/**
* 标准化总占地面积输出
*/
public static String formatAllArea(String all_area) {
if (StringUtils.isNotBlank(all_area)) {
Double formattedVal = 0.0;
if (all_area.toString().indexOf("公顷") != -1) {
String val = all_area.toString();
formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000);
} else if(checkValue(all_area)!=null) {
formattedVal = Double.valueOf(all_area);
}
if (formattedVal > 100000 || formattedVal == 0.0) {
return null;
}
return String.valueOf(formattedVal);
}
return null;
}
/**
* 提取浮点数与整数
*/
static String getNumber(String input) {
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
}
return null;
}
static Float checkValue(String value){
if(StringUtils.isNotBlank(value)){
value = getNumber(value);
if(StringUtils.isNotBlank(value)){
value = subZeroAndDot(value);
if(isNumber(value)){
Float v = Float.parseFloat(value);
if(v>0){
return v;
}else{
return null;
}
}else if(isFloat(value)){
Float v = Float.parseFloat(value);
if(v>0){
return v;
}else{
return null;
}
} else{
return null;
}
}
}
return null;
}
static boolean isNumber(String str){
try {
Integer.parseInt(str);
} catch (NumberFormatException e) {
return false;
}
return true;
}
static boolean isFloat(String str){
try {
Float.parseFloat(str);
} catch (NumberFormatException e) {
return false;
}
return true;
}
/**
* 正则表达式去掉多余的.与0
* @param
* @return
*/
static String subZeroAndDot(String v){
if(StringUtils.isNotBlank(v)){
if(v.indexOf(".")!=-1){
Double myNum2 = Double.valueOf(v);
BigDecimal b = new java.math.BigDecimal(myNum2);
Double myNum3 = b.setScale(2, java.math.BigDecimal.ROUND_HALF_UP).doubleValue();
v = myNum3.toString();
}
if(v.indexOf(".")>-1){
while(true)
{
if(v.endsWith("0")){
v = v.substring(0,v.length()-1);
}else{
break;
}
}
if(v.endsWith(".")){
v=v.substring(0,v.length()-1);
}
}
}
return v;
}
}
@@ -0,0 +1,7 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.BsEvaluation;
public interface BsEvaluationDao extends MongoDao<BsEvaluation> {
}
@@ -0,0 +1,8 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.model.BsEvaluation;
public class BsEvaluationDaoImpl extends MongoDaoImpl<BsEvaluation> implements
BsEvaluationDao {
}
@@ -99,6 +99,9 @@ public class PaiMai implements MongoEntity {
private Integer marketAvgPrice;
/**物业类型*/
private String matchEstateType;
private String houseNum;
private String unitNum;
private String roomNum;
@Id
@GeneratedValue
@@ -488,4 +491,28 @@ public class PaiMai implements MongoEntity {
public void setMatchEstateType(String matchEstateType) {
this.matchEstateType = matchEstateType;
}
public String getHouseNum() {
return houseNum;
}
public void setHouseNum(String houseNum) {
this.houseNum = houseNum;
}
public String getUnitNum() {
return unitNum;
}
public void setUnitNum(String unitNum) {
this.unitNum = unitNum;
}
public String getRoomNum() {
return roomNum;
}
public void setRoomNum(String roomNum) {
this.roomNum = roomNum;
}
}
@@ -0,0 +1,184 @@
package com.fangjia.dal.mongo.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
import java.util.Date;
@Entity
@Table(name = "business.evaluation")
public class BsEvaluation implements MongoEntity {
private String id;
private String fullAddress;
private String commid;
private String cityName;
private String distName;
private String commName;
private String address;
private Double buildArea;
private Integer floor;
private Integer floorcnt;
private Integer roomcnt;
private Integer livingroomcnt;
private String houseaspect;//ÄÏ
private String fitment;
private Date cDate;
private Integer unitPrice;
private Integer totalPrice;
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@Column(name = "_id")
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "full_address")
public String getFullAddress() {
return fullAddress;
}
public void setFullAddress(String fullAddress) {
this.fullAddress = fullAddress;
}
@Column(name = "commid")
public String getCommid() {
return commid;
}
public void setCommid(String commid) {
this.commid = commid;
}
@Column(name = "cityname")
public String getCityName() {
return cityName;
}
public void setCityName(String cityName) {
this.cityName = cityName;
}
@Column(name = "distname")
public String getDistName() {
return distName;
}
public void setDistName(String distName) {
this.distName = distName;
}
@Column(name = "commname")
public String getCommName() {
return commName;
}
public void setCommName(String commName) {
this.commName = commName;
}
@Column(name = "address")
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
@Column(name = "buildarea")
public Double getBuildArea() {
return buildArea;
}
public void setBuildArea(Double buildArea) {
this.buildArea = buildArea;
}
@Column(name = "floor")
public Integer getFloor() {
return floor;
}
public void setFloor(Integer floor) {
this.floor = floor;
}
@Column(name = "floorcnt")
public Integer getFloorcnt() {
return floorcnt;
}
public void setFloorcnt(Integer floorcnt) {
this.floorcnt = floorcnt;
}
@Column(name = "roomcnt")
public Integer getRoomcnt() {
return roomcnt;
}
public void setRoomcnt(Integer roomcnt) {
this.roomcnt = roomcnt;
}
@Column(name = "livingroomcnt")
public Integer getLivingroomcnt() {
return livingroomcnt;
}
public void setLivingroomcnt(Integer livingroomcnt) {
this.livingroomcnt = livingroomcnt;
}
@Column(name = "houseaspect")
public String getHouseaspect() {
return houseaspect;
}
public void setHouseaspect(String houseaspect) {
this.houseaspect = houseaspect;
}
@Column(name = "fitment")
public String getFitment() {
return fitment;
}
public void setFitment(String fitment) {
this.fitment = fitment;
}
@Column(name = "c_date")
public Date getcDate() {
return cDate;
}
public void setcDate(Date cDate) {
this.cDate = cDate;
}
@Column(name = "unitprice")
public Integer getUnitPrice() {
return unitPrice;
}
public void setUnitPrice(Integer unitPrice) {
this.unitPrice = unitPrice;
}
@Column(name = "totalprice")
public Integer getTotalPrice() {
return totalPrice;
}
public void setTotalPrice(Integer totalPrice) {
this.totalPrice = totalPrice;
}
}
@@ -63,6 +63,7 @@ public class DistrictFacilitys implements MongoEntity {
private String imgUrl3D;
private List<String> school;
private List<String> keySchool;
@Id
@GeneratedValue
@@ -380,4 +381,12 @@ public class DistrictFacilitys implements MongoEntity {
this.school = school;
}
@Column(name = "key_school")
public List<String> getKeySchool() {
return keySchool;
}
public void setKeySchool(List<String> keySchool) {
this.keySchool = keySchool;
}
}
@@ -48,6 +48,11 @@ public class DealPrice implements MongoEntity {
private Integer floor;//当前楼层
private Long score;
private String matchId;
private String matchCity;
private String matchRegion;
private String matchName;
@Id
@GeneratedValue
@Column(name = "_id", unique = true, nullable = false)
@@ -296,4 +301,40 @@ public class DealPrice implements MongoEntity {
}
return true;
}
@Column(name = "match_id")
public String getMatchId() {
return matchId;
}
public void setMatchId(String matchId) {
this.matchId = matchId;
}
@Column(name = "match_city")
public String getMatchCity() {
return matchCity;
}
public void setMatchCity(String matchCity) {
this.matchCity = matchCity;
}
@Column(name = "match_region")
public String getMatchRegion() {
return matchRegion;
}
public void setMatchRegion(String matchRegion) {
this.matchRegion = matchRegion;
}
@Column(name = "match_name")
public String getMatchName() {
return matchName;
}
public void setMatchName(String matchName) {
this.matchName = matchName;
}
}
@@ -448,6 +448,7 @@ public class HouseLoudong extends com.fangjia.dal.mongo.model.HouseLoudong{
this.openTotalPrice = openTotalPrice;
}
@Column(name = "district_id")
public String getDistrictId() {
return districtId;
}
@@ -101,7 +101,11 @@ public class Loudong implements MongoEntity {
* seaweed中的小区名
*/
private String nameInSeaweed;
/**
* seaweed中的小区id
*/
private String idInSeaweed;
@Column(name = "remark")
public String getRemark() {
return remark;
@@ -402,4 +406,13 @@ public class Loudong implements MongoEntity {
public void setNameInSeaweed(String nameInSeaweed) {
this.nameInSeaweed = nameInSeaweed;
}
@Column(name = "id_in_seaweed")
public String getIdInSeaweed() {
return idInSeaweed;
}
public void setIdInSeaweed(String idInSeaweed) {
this.idInSeaweed = idInSeaweed;
}
}
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.open.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityBasePriceDao extends MongoDao<CommunityBasePrice> {
}
@@ -0,0 +1,9 @@
package com.fangjia.dal.mongo.open.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.CountDistrictStatBycity;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public interface CommunityPoiDao extends MongoDao<CommunityPoi> {
}
@@ -0,0 +1,10 @@
package com.fangjia.dal.mongo.open.dao.impl;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
public class CommunityBasePriceDaoImpl extends MongoDaoImpl<CommunityBasePrice> implements CommunityBasePriceDao {
}
@@ -0,0 +1,10 @@
package com.fangjia.dal.mongo.open.dao.impl;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.open.dao.ApiVisitCountDao;
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.dal.mongo.tool.model.ApiVisitCount;
public class CommunityPoiDaoImpl extends MongoDaoImpl<CommunityPoi> implements CommunityPoiDao {
}
@@ -0,0 +1,74 @@
package com.fangjia.dal.mongo.open.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.Id;
import javax.persistence.Table;
/**
* Ð¡Çø½¨ÐÐÖÃÐŶÈÐÅÏ¢
*
* @author shenjian@fangjia.com
*/
@Entity
@Table(name = "business.community_base_price_gj")
public class CommunityBasePrice implements MongoEntity {
private static final long serialVersionUID = -8010010773207622878L;
private String id;
private String city;
private String region;
private String name;
private Double credit;
@Id
@Column(name = "_id")
@Override
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "credit")
public Double getCredit() {
return credit;
}
public void setCredit(Double credit) {
this.credit = credit;
}
}
@@ -0,0 +1,135 @@
package com.fangjia.dal.mongo.open.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
import java.util.Date;
/**
* 小区周边配套统计信息
*
* @author shenjian@fangjia.com
*/
@Entity
@Table(name = "fangjia_stat.community_poi_gj")
public class CommunityPoi implements MongoEntity {
private static final long serialVersionUID = -7663644539542318409L;
private String id;
private String city;
private String region;
private String name;
private Integer subway;
private Integer traffic;
private Integer bank;
private Integer primary;
private Integer junior;
private Integer hospital;
/**
* 不利设施,丧葬
*/
private Integer funeral;
@Id
@Column(name = "_id")
@Override
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "subway")
public Integer getSubway() {
return subway;
}
public void setSubway(Integer subway) {
this.subway = subway;
}
@Column(name = "traffic")
public Integer getTraffic() {
return traffic;
}
public void setTraffic(Integer traffic) {
this.traffic = traffic;
}
@Column(name = "bank")
public Integer getBank() {
return bank;
}
public void setBank(Integer bank) {
this.bank = bank;
}
@Column(name = "primary")
public Integer getPrimary() {
return primary;
}
public void setPrimary(Integer primary) {
this.primary = primary;
}
@Column(name = "junior")
public Integer getJunior() {
return junior;
}
public void setJunior(Integer junior) {
this.junior = junior;
}
@Column(name = "hospital")
public Integer getHospital() {
return hospital;
}
public void setHospital(Integer hospital) {
this.hospital = hospital;
}
@Column(name = "funeral")
public Integer getFuneral() {
return funeral;
}
public void setFuneral(Integer funeral) {
this.funeral = funeral;
}
}
@@ -67,6 +67,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
must.add(addTerms("district_name", alias));
must.add(addMatch("cat", "sell"));
must.add(addRange("avg_price", minPrice, maxPrice));
must.add(addRangeGte("p_date", 20210501));
bool.put("must", must);
query.put("bool", bool);
reqBody.put("query", query);
@@ -161,6 +163,16 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
return term;
}
private JSONObject addRangeGte(String name, Number min){
JSONObject term = new JSONObject();
JSONObject range = new JSONObject();
JSONObject rangeValue = new JSONObject();
rangeValue.put("gte", min);
range.put(name, rangeValue);
term.put("range", range);
return term;
}
private List<Seawater> wrapSeawater(List<JSONObject> jsonList){
List<Seawater> seawaterList= new ArrayList<Seawater>();
@@ -203,6 +215,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
seawater.setFloor(source.getInt("floor"));
}catch(Exception e){
seawater.setFloor(0);
}
try{
seawater.setFloorInfo(source.getString("floor_info"));
}catch(Exception e){
}
if(source.containsKey("fitment")){
seawater.setFitment(source.getString("fitment"));
@@ -231,6 +248,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
if(source.containsKey("root")){
seawater.setRoot(source.getString("root"));
}
if(source.containsKey("estate_type")){
seawater.setEstateType(source.getString("estate_type"));
}
if(source.containsKey("build_type")){
seawater.setBuildType(source.getString("build_type"));
}
seawaterList.add(seawater);
}
return seawaterList;
@@ -379,11 +404,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
}
// ²»Õ¹Ê¾É¾³ý
JSONObject param = new JSONObject();
param.put("status", 0);
JSONObject temp = new JSONObject();
temp.put("term", param);
jsonList.add(temp);
// JSONObject param = new JSONObject();
// param.put("status", 0);
// JSONObject temp = new JSONObject();
// temp.put("term", param);
// jsonList.add(temp);
query.put("must", jsonList);
q.put("bool", query);
@@ -48,6 +48,7 @@ public class Seawater implements SeawaterEntity {
private Double area;
private Integer height;
private Integer floor;
private String floorInfo;
private String fitment;
private Integer room;
private Integer hall;
@@ -58,6 +59,9 @@ public class Seawater implements SeawaterEntity {
private String url;
private String root;
private Integer avgPrice;
private String purpose;
private String estateType;
private String buildType;
@Id
@Column(name = "_id")
@@ -195,6 +199,15 @@ public class Seawater implements SeawaterEntity {
this.floor = floor;
}
@Column(name = "floorInfo")
public String getFloorInfo() {
return floorInfo;
}
public void setFloorInfo(String floorInfo) {
this.floorInfo = floorInfo;
}
@Column(name = "fitment")
public String getFitment() {
return fitment;
@@ -285,4 +298,31 @@ public class Seawater implements SeawaterEntity {
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
@Column(name = "purpose")
public String getPurpose() {
return purpose;
}
public void setPurpose(String purpose) {
this.purpose = purpose;
}
@Column(name = "estate_type")
public String getEstateType() {
return estateType;
}
public void setEstateType(String estateType) {
this.estateType = estateType;
}
@Column(name = "build_type")
public String getBuildType() {
return buildType;
}
public void setBuildType(String buildType) {
this.buildType = buildType;
}
}
@@ -1,6 +1,7 @@
package com.fangjia.open.controllers;
import com.fangjia.core.util.DateUtil;
import com.fangjia.core.util.HttpRequest;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.SortParam;
import com.fangjia.dal.common.DistrictKey;
@@ -14,6 +15,7 @@ import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.service.fsh.DistrictNewPriceService;
import com.fangjia.service.open.PaiMaiService;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.bson.types.ObjectId;
@@ -23,13 +25,14 @@ import java.util.regex.Pattern;
/**
* 法拍数据提供接口
*/
@Authority(role = Role.USER)
//@Authority(role = Role.USER)
public class AuctionController {
private PaiMaiService paiMaiService;
private DistrictNewPriceService districtNewPriceService;
private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType"};
private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url",
"matchEstateType","type","count", "tradeAvgPrice", "evaluationPrice"};
public Object housesAction(AuctionModel model){
RestfulModelAndView mav = new RestfulModelAndView();
@@ -187,6 +190,79 @@ public class AuctionController {
return mav;
}
public Object searchPaimaisAction(AuctionModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
List<QueryParam> params = new ArrayList<QueryParam>();
if(StringUtils.isNotBlank(model.getCity())){
params.add(new QueryParam("matchCity", model.getCity()));
}
if(StringUtils.isNotBlank(model.getDistrict())){
params.add(new QueryParam("matchRegion", model.getDistrict()));
}
if(StringUtils.isNotBlank(model.getName())){
params.add(new QueryParam("matchName", model.getName()));
}
Date startDate = null;
if(StringUtils.isNotBlank(model.getStartDate())){
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
if(startDate != null){
params.add(new QueryParam("startDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
}
}
List<SortParam> sortParams = new ArrayList<SortParam>();
sortParams.add(new SortParam("startDate", false));
List<PaiMai> paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS);
if(paimais!=null && paimais.size()>0){
for(PaiMai paiMai : paimais){
Map<String, Object> map = standard(paiMai.getCity(), paiMai.getAddress());
if(map!=null){
if (map.containsKey("buildingNumber")) {
paiMai.setHouseNum((String) map.get("buildingNumber"));
}
if (map.containsKey("unitNumber")) {
paiMai.setUnitNum((String) map.get("unitNumber"));
}
if (map.containsKey("roomNumber")) {
paiMai.setRoomNum((String) map.get("roomNumber"));
}
}
}
}
mav.setModel(BasicModel.buildOKModel().setResult(paimais));
return mav;
}
private Map standard(String city, String address){
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
try{
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
} catch(Exception e){
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
} catch(Exception ex){
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
if(json.containsKey("result")){
return (Map)json.get("result");
}
}
return null;
}
public void setPaiMaiService(PaiMaiService paiMaiService) {
this.paiMaiService = paiMaiService;
}
@@ -15,8 +15,10 @@ import java.util.Map.Entry;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.mongo.model.Metro;
import com.fangjia.open.model.BasicModel;
import com.fangjia.service.bo.DistrictStatBo;
import org.apache.commons.lang.StringUtils;
import com.fangjia.core.util.DateUtil;
@@ -211,6 +213,42 @@ public class DealPriceController {
mv.setModel(resMap);
return mv;
}
public Object searchListAction(final DealPriceModel model) {
RestfulModelAndView mv = new RestfulModelAndView();
mv.setViewType("rest");
if (StringUtils.isBlank(model.getId())) {
mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空"));
return mv;
}
District district = districtService.findDistrictById(model.getId());
DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
if(district == null){
mv.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
return mv;
}
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
Integer minPrice = null;
Integer maxPrice = null;
if(bo.getDistrictStat()!=null){
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
}
Date startDate = null;
Date endDate = null;
if(StringUtils.isNotBlank(model.getStartDate())){
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
}
if(StringUtils.isNotBlank(model.getEndDate())){
endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd");
}
List<DealPrice> dealPrices = dealPriceService.getNearlySameDealPrice(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit());
mv.setModel(BasicModel.buildOKModel().setResult(dealPrices));
return mv;
}
private List<DealPrice> findDealPricesByName(String city,String region ,String name){
Calendar calendar = Calendar.getInstance();
@@ -7,11 +7,11 @@ import javax.servlet.http.HttpServletRequest;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.DateUtil;
import com.fangjia.dal.mongo.model.RegionStat;
import com.fangjia.dal.mongo.model.RegionStatRent;
import com.fangjia.dal.mongo.model.*;
import com.fangjia.open.model.AvgPriceQueryModel;
import com.fangjia.open.model.BasicModel;
import com.fangjia.service.datastore.RegionStatService;
import com.fangjia.service.district.DistrictFacilitysService;
import com.fangjia.service.district.DistrictStatRentService;
import com.fangjia.service.district.RegionStatRentService;
import com.fangjia.service.mongo.fangjia.AverageAnalyService;
@@ -38,8 +38,6 @@ import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.common.FangjiaServerAreaContextHolder;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.DistrictHouse;
import com.fangjia.dal.mongo.model.DistrictStat;
import com.fangjia.dal.mongo.model.bkoff.ExtractDistrict;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.dal.sql.model.ApiUser;
@@ -93,6 +91,7 @@ public class DistrictController {
private RegionBlockService regionBlockService;
private RegionStatService regionStatService;
private RegionStatRentService regionStatRentService;
private DistrictFacilitysService districtFacilitysService;
public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea",
@@ -444,19 +443,19 @@ public class DistrictController {
*
* @return
*/
public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
@Param(name = "name") String name, HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
return mav;
} else {
DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name));
if (null == districtBo) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件"));
public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
@Param(name = "name") String name, HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
return mav;
}
} else {
DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name));
if (null == districtBo) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件"));
return mav;
}
districtBo.setStat(this.districtStatService.findDistrictStat(districtBo.getDistrictKey()));
// 去除小区描述中的html标签
districtBo.setDescription(DataUtil.removeSpecialString(districtBo.getDescription()));
@@ -1146,6 +1145,25 @@ public class DistrictController {
ret.put("RESULT", 200);
return mav;
}
public Object facilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
@Param(name = "name") String name, HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
return mav;
} else {
District districtBo = districtService.findDistrict(new DistrictKey(city, region, name), "district");
if (null == districtBo) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在"));
return mav;
}
DistrictFacilitys facilitys = districtFacilitysService.findByDistrict(districtBo);
mav.setModel(BasicModel.buildOKModel().setResult(facilitys));
}
return mav;
}
@@ -1230,4 +1248,7 @@ public class DistrictController {
this.regionStatRentService = regionStatRentService;
}
public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) {
this.districtFacilitysService = districtFacilitysService;
}
}
@@ -10,9 +10,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import com.fangjia.dal.mongo.model.BsEvaluation;
import com.fangjia.dal.mongo.model.bkoff.DealPrice;
import com.fangjia.service.bo.AddrEvaResult;
import com.fangjia.service.bo.DistrictStatBo;
import com.fangjia.service.bo.EvaluationBo;
import com.fangjia.service.bo.HouseScorer;
import com.fangjia.service.datastore.*;
import com.fangjia.service.district.DistrictStatService;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
@@ -54,11 +59,6 @@ import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.service.api.AutoCompleteService;
import com.fangjia.service.api.KnowbaseService;
import com.fangjia.service.common.SellSimilar;
import com.fangjia.service.datastore.BusinessEstateService;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.datastore.EvaluationReportService;
import com.fangjia.service.datastore.EvaluationService;
import com.fangjia.service.datastore.HouseService;
import com.fangjia.service.sql.ApiUserService;
import com.fangjia.service.sql.UserService;
import com.fangjia.service.web.AddressService;
@@ -94,6 +94,8 @@ public class EvaluationController {
private ApiUserService apiUserService;
private UserService userService;
private AddressService addressService;
private DistrictStatService districtStatService;
private BsEvaluationService bsEvaluationService;
private final static String AK = OpenConfig.getConfig().getBaiduAk();
public Object lngLatEvaluateAction(EvaluationModel model) {
@@ -1005,6 +1007,44 @@ public class EvaluationController {
return mav;
}
public Object bsListAction(EvaluationModel model) {
RestfulModelAndView mv = new RestfulModelAndView();
mv.setViewType("rest");
if (StringUtils.isBlank(model.getId())) {
mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空"));
return mv;
}
District district = districtService.findDistrictById(model.getId());
DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
if(district == null){
mv.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
return mv;
}
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
Integer minPrice = null;
Integer maxPrice = null;
if(bo.getDistrictStat()!=null){
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
}
Date startDate = null;
Date endDate = null;
if(StringUtils.isNotBlank(model.getStartDate())){
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
}
if(StringUtils.isNotBlank(model.getEndDate())){
endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd");
}
List<BsEvaluation> dealPrices = bsEvaluationService.getNearlySameEvaluation(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit());
mv.setModel(BasicModel.buildOKModel().setResult(dealPrices));
return mv;
}
public void setEvaluationService(EvaluationService evaluationService) {
this.evaluationService = evaluationService;
}
@@ -1017,6 +1057,18 @@ public class EvaluationController {
this.knowbaseService = knowbaseService;
}
public void setDistrictStatService(DistrictStatService districtStatService) {
this.districtStatService = districtStatService;
}
public void setBsEvaluationService(BsEvaluationService bsEvaluationService) {
this.bsEvaluationService = bsEvaluationService;
}
public void setAutoCompleteService(AutoCompleteService autoCompleteService) {
this.autoCompleteService = autoCompleteService;
}
@@ -11,6 +11,8 @@ import com.fangjia.dal.mongo.model.DistrictFacilitys;
import com.fangjia.dal.mongo.model.FangjiaOfficialAreaMap;
import com.fangjia.dal.mongo.model.Metro;
import com.fangjia.dal.mongo.model.StatsIndex;
import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.dal.sql.model.ApiUser;
import com.fangjia.datastore.driver.FacetResult;
@@ -29,6 +31,7 @@ import com.fangjia.open.ws.query.CollateralSampleQuery;
import com.fangjia.open.ws.query.ShopCollateralSampleQuery;
import com.fangjia.service.address.AddressParser;
import com.fangjia.service.bo.Collateral;
import com.fangjia.service.bo.DistrictStatBo;
import com.fangjia.service.bo.Sample;
import com.fangjia.service.bo.ShopSample;
import com.fangjia.service.datastore.CzbankSampleService;
@@ -39,11 +42,13 @@ import com.fangjia.service.datastore.bean.CollateralSampleQueryBean;
import com.fangjia.service.datastore.bean.ShopCollateralSampleQueryBean;
import com.fangjia.service.district.DistrictFacilitysService;
import com.fangjia.service.district.DistrictStatService;
import com.fangjia.service.district.SqmetreRentService;
import com.fangjia.service.menu.Item;
import com.fangjia.service.menu.MenuCategoryEnum;
import com.fangjia.service.menu.Value;
import com.fangjia.service.mongo.fangjia.DealPriceService;
import com.fangjia.service.mongo.fangjia.MetroService;
import com.fangjia.service.mongo.fangjia.SeawaterService;
import com.fangjia.service.mongo.fangjia.StatsIndexService;
import com.fangjia.service.open.FangjiaOfficialAreaService;
import com.fangjia.service.open.HousePojo;
@@ -86,6 +91,9 @@ public class HouseController {
private CzbankSampleService czbankSampleService;
private DistrictService districtService;
private FangjiaOfficialAreaService fangjiaOfficialAreaService;
private SeawaterService seawaterService;
private SeawaterDao seawaterDao;
private List<String> PROPERTY_TYPE_CODE_LIST = Arrays.asList("01", "02");
@@ -399,6 +407,51 @@ public class HouseController {
}
}
public Object listAction(final HouseSearchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) {
mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项"));
return mav;
}
District district = null;
DistrictKey districtKey = null;
if(StringUtils.isNotBlank(model.getName())){
districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName());
district = districtService.findDistrict(districtKey, "district");
}else{
district = districtService.findDistrictById(model.getId());
if(district!=null){
districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
}
}
if(district == null){
mav.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
return mav;
}
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
Integer minPrice = null;
Integer maxPrice = null;
if(bo.getDistrictStat()!=null){
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
}
List<String> names = new ArrayList<String>();
names.add(district.getName());
if(district.getAlias() != null){
for(String alia : district.getAlias()){
if(StringUtils.isNotBlank(alia) && !names.contains(alia)){
names.add(alia);
}
}
}
List<Seawater> houses = seawaterDao.findSeawatersFromEs("sell", district.getCity(), district.getRegion(), district.getName(), district.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater");
//List<Seawater> houses1 = seawaterDao.findHouses(district.getCity(), district.getRegion(), names, minPrice, maxPrice, model.getPageSize());
mav.setModel(BasicModel.buildOKModel().setResult(houses));
return mav;
}
String getPicUrl(String url, List<String> urls) {
if (!urls.contains(url)) {
urls.add(url);
@@ -832,6 +885,13 @@ public class HouseController {
return ret;
}
public void setSeawaterService(SeawaterService seawaterService) {
this.seawaterService = seawaterService;
}
public void setSeawaterDao(SeawaterDao seawaterDao) {
this.seawaterDao = seawaterDao;
}
private boolean checkExists(FacetQueryResult<HousePojo> houseBoList, HousePojo house) {
for (HousePojo bo : houseBoList) {
@@ -31,12 +31,15 @@ import javax.validation.Valid;
import com.fangjia.core.util.*;
import com.fangjia.dal.*;
import com.fangjia.dal.mongo.model.*;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.open.dto.*;
import com.fangjia.service.district.*;
import com.fangjia.service.mongo.fangjia.*;
import com.fangjia.service.open.*;
import com.fangjia.service.open.impl.CommunityPoiServiceImpl;
import com.fangjia.service.util.DistrictScoreUtil;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
@@ -175,6 +178,9 @@ public class PropertyController {
private SeawaterDao seawaterDao;
private SeawaterService seawaterService;
private SqmetreRentService sqmetreRentService;
private CommunityPoiService communityPoiService;
private CommunityBasePriceService communityBasePriceService;
private EvaluateFactorService evaluateFactorService;
private FileDao fileDao = FileDao.getInstance();
private static final int DEFAULT_PAGE_SIZE = 20;
@@ -390,10 +396,31 @@ public class PropertyController {
put("5000", "five");// 五线
}
};
/**
* 第三方均价映射
*/
private static final Map<String, String> OTHER_PRICE_MAP = new HashMap<String, String>() {
{
put("zhuge_price", "诸葛找房");
put("anjuke_price", "安居客");
put("fang_price", "房天下");
put("lianjia_price", "链家");
put("cityhouse_price", "禧泰数据");
put("maitian_price", "麦田");
put("ke_price", "贝壳");
put("taofang_price", "淘房365");
put("zhongyuan_price", "中原");
put("tbs_price", "兔博士");
put("qfang_price", "Q房");
put("fangtan_price", "房探");
put("fangduoduo_price", "房多多");
put("wiwj_price", "我爱我家");
}
};
/**
* 利用百度API把GPS转换成Google经纬度
*
*
* @param lng
* @param lat
* @return
@@ -415,7 +442,7 @@ public class PropertyController {
/**
* 利用算法模板把GPS转换成Google经纬度
*
*
* @param lng
* @param lat
* @return
@@ -607,7 +634,7 @@ public class PropertyController {
/**
* 判断小区是否有经纬度
*
*
* @param district
* @return
*/
@@ -618,7 +645,7 @@ public class PropertyController {
/**
* 调用匹配接口匹配小区
*
*
* @param city
* @param address
* @param region
@@ -666,17 +693,6 @@ public class PropertyController {
}
String userId = FangjiaApiUtil.parseUserId(model.getToken());
/**超级用户和上海交大账户不控制权限*/
if (!"1007497".equals(userId) && !"1007632".equals(userId) && !"1007548".equals(userId)
&& !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId)
&& !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId)
&& !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId)
&& !"1007747".equals(userId) && !"1007757".equals(userId)
&& model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制
mav.setModel(BasicModel.buildUnAuthorizedModel());
return mav;
}
District district = null;
if (StringUtils.isNotBlank(model.getId())){
if(ObjectId.isValid(model.getId())){
@@ -740,18 +756,18 @@ public class PropertyController {
} else {
districtInfoDto.setBuildingType("");
}
districtInfoDto.setAlias(district.getAlias());
districtInfoDto.setAlias(district.getMappingAlias());
districtInfoDto.setAddressAlias(district.getAddrAlias());
districtInfoDto.setCommunity(district.getStreetCommunity());
districtInfoDto.setCommercialArea(district.getBusinessQuarter());
if (district.getHouseholdCount() != null) {
districtInfoDto.setTotalUnitCount(district.getHouseholdCount().toString());
districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString()));
} else {
districtInfoDto.setTotalUnitCount("");
}
districtInfoDto.setManagementFee(district.getEstateCharge());
if (district.getCarSiteCount() != null) {
districtInfoDto.setParking(district.getCarSiteCount().toString());
districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString()));
} else {
districtInfoDto.setParking("");
}
@@ -772,10 +788,10 @@ public class PropertyController {
districtInfoDto.setFinishBuildTime(district.getFinishBuildingDate());
districtInfoDto.setMoveinTime(district.getLiveDate());
districtInfoDto.setFloorSpace(district.getAllArea());
districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea()));
districtInfoDto.setConstructionSpace(district.getBuildArea());
districtInfoDto.setPlotRatio(district.getCubagePercent());
districtInfoDto.setGreenPercentage(district.getVirescencePercent());
districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent()));
districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent()));
districtInfoDto.setPreSalePermit(district.getLicence());
districtInfoDto.setDeveloper(district.getDeveloper());
districtInfoDto.setDiscount(district.getDiscount());
@@ -792,6 +808,7 @@ public class PropertyController {
districtInfoDto.setDescription(district.getDescription());
districtInfoDto.setSalesOfficeAdd(district.getSellAddress());
districtInfoDto.setSalesPhoneNumber(district.getMobile());
districtInfoDto.setTownship(district.getTownship());
if (district.getLoudongCount() != null && district.getLoudongCount() > 0) {
districtInfoDto.setBuildingCount(district.getLoudongCount());
}
@@ -960,6 +977,12 @@ public class PropertyController {
return mav;
}
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
if(loudong != null && loudong.getFloors() >= model.getFloor()){
model.setMaxFloor(loudong.getFloors());
}
}
model.setRegion(district.getRegion());
model.setDistrictName(district.getName());
if (StringUtils.isBlank(model.getPropertyType())) {
@@ -996,6 +1019,32 @@ public class PropertyController {
}
}
// 调整面积不同面积价格
try {
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
int orgAvgPrice = 0;
if (districtStatBo != null) {
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
} else if (districtStatBo.getDistrictStat() != null) {
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
}
}
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
}
} catch (Exception ex) {
}
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
@@ -1116,7 +1165,7 @@ public class PropertyController {
model.setCalDate(calDate);
}
/* 先到CPDB库中查找小区 */
District district = matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName());
District district = districtService.findDistrict(new DistrictKey(model.getCity(), model.getDistrict(), model.getName()), "district");//matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName());
if(district == null){
//防止地址模糊,名称详细的情况
district = matchDistrict(model.getCity(), model.getName(), model.getDistrict(), model.getAddress());
@@ -1166,19 +1215,52 @@ public class PropertyController {
evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
return mav;
}
// 调整面积不同面积价格
try {
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
int orgAvgPrice = 0;
if (districtStatBo != null) {
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
} else if (districtStatBo.getDistrictStat() != null) {
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
}
}
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
}
} catch (Exception ex) {
}
Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>();
districtPriceMap.put("city", district.getCity());
districtPriceMap.put("district", district.getRegion());
districtPriceMap.put("name", district.getName());
long totalPrice = evaluation.getBigTotalPrice();
if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
evaluation.setBigTotalPrice(totalPrice);
evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
}
// 取消系数下调
// if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
// totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
// evaluation.setBigTotalPrice(totalPrice);
// evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
// }
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
// 新增置信度
CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId());
if (communityBasePrice != null) {
districtPriceMap.put("credit", communityBasePrice.getCredit());
} else {
districtPriceMap.put("credit", evaluation.getCredit());
}
List<String> names = new ArrayList<String>();
names.add(district.getName());
if(district.getAlias() != null){
@@ -1190,6 +1272,31 @@ public class PropertyController {
}
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9);
Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1);
// 新增第三方均价
try {
double dCredit = 0.61;
Map<String, Object> rMap = getLastEvaluationPrice(district, evaluation, dCredit);
if (null != rMap) {
if (rMap.containsKey("other_price")) {
Map<String, Integer> otherPriceMap = (Map<String, Integer>) rMap.get("other_price");
Map<String, Integer> priceResultMap = new HashMap<String, Integer>();
for (String key : otherPriceMap.keySet()) {
Integer price = otherPriceMap.get(key);
// 剔除偏差大于10的
if (min10Price <= price && max10Price >= price) {
if (OTHER_PRICE_MAP.containsKey(key)) {
priceResultMap.put(OTHER_PRICE_MAP.get(key), price);
}
}
}
districtPriceMap.put("otherPrice", priceResultMap);
}
}
} catch (Exception ex) {
}
List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
if(houses.size() > 3){
districtPriceMap.put("calcType", 1);
@@ -1218,7 +1325,7 @@ public class PropertyController {
}else if(district.getLng() != null && district.getLng() > 0){
double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6);
double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6);
List<District> districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 4, null);
List<District> districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 6, null);
if(districts.size() < 2){
evaluationService.remove(evaluation.getId());
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
@@ -1256,9 +1363,15 @@ public class PropertyController {
districtDto.append("buildYear", String.valueOf(buildYear));
}
districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d));
if(districtAnalyData != null && districtAnalyData.getHalfYearMarkup() != null){
districtDto.append("halfYearMarkup",DataUtil.roundDouble( districtAnalyData.getHalfYearMarkup() * 100 ,2));
if(districtAnalyData != null){
if (districtAnalyData.getHalfYearMarkup() != null) {
districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2));
}
if (districtAnalyData.getAvgPrice() != null && minPrice <= districtAnalyData.getAvgPrice() && maxPrice >= districtAnalyData.getAvgPrice()) {
districtDto.append("avgPrice", districtAnalyData.getAvgPrice());
}
}
nearCommunityDtos.add(districtDto);
}
Integer buildYear = districtService.getBuildYear(district);
@@ -1266,6 +1379,18 @@ public class PropertyController {
modelFeaturesFto.put("buildYear", String.valueOf(buildYear));
}
modelFeaturesFto.append("nearCommunity", nearCommunityDtos);
// 模型特征新增周边数量
CommunityPoi communityPoi = communityPoiService.getById(district.getId());
if (communityPoi != null) {
modelFeaturesFto.put("subway", communityPoi.getSubway());
modelFeaturesFto.put("traffic", communityPoi.getTraffic());
modelFeaturesFto.put("bank", communityPoi.getBank());
modelFeaturesFto.put("primary", communityPoi.getPrimary());
modelFeaturesFto.put("hospital", communityPoi.getHospital());
modelFeaturesFto.put("junior", communityPoi.getJunior());
modelFeaturesFto.put("funeral", communityPoi.getFuneral());
}
districtPriceMap.put("modelFeatures", modelFeaturesFto);
}
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
@@ -1473,7 +1598,7 @@ public class PropertyController {
return mav;
}
}
/**
* 四川锦程 定制接口 多建筑年代
* @param model
@@ -1585,7 +1710,7 @@ public class PropertyController {
districtPriceMap.put("buildYear", matcher3.group());
}
}
// 根据城市等 级设置D级可信度
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
@@ -2098,7 +2223,7 @@ public class PropertyController {
/**
* 小区租金查询接口
*
*
* @param model
* @return
*/
@@ -2577,7 +2702,7 @@ public class PropertyController {
/**
* 批量获取小区的均价,均价走势数据
*
*
* @param request
* @return
*/
@@ -2719,7 +2844,7 @@ public class PropertyController {
/**
* 批量获取小区的配套数据,最多12个小区
*
*
* @param request
* @return
*/
@@ -2864,7 +2989,7 @@ public class PropertyController {
/**
* 获取JSON参数中的num参数值
*
*
* @param jo
* @return
*/
@@ -2889,7 +3014,7 @@ public class PropertyController {
/**
* 获取JSON参数中的radius值
*
*
* @param jo
* @return
*/
@@ -3697,7 +3822,7 @@ public class PropertyController {
/**
* 解析出关键字中的小区名
*
*
* @param keyword
* @return
*/
@@ -3814,7 +3939,7 @@ public class PropertyController {
/**
* 小区统计分析接口
*
*
* @param model
* @return
*/
@@ -3899,7 +4024,7 @@ public class PropertyController {
/**
* 小区交通接口
*
*
* @param model
* @return
*/
@@ -3957,7 +4082,7 @@ public class PropertyController {
/**
* 小区周边接口
*
*
* @param model
* @return
*/
@@ -4016,7 +4141,7 @@ public class PropertyController {
/**
* 相同小区相似户型房源挂牌量接口<br>
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
*
*
* @author yinjihuan
* @date 2016年9月1日
* @param city
@@ -4056,7 +4181,7 @@ public class PropertyController {
/**
* 周边小区相似户型挂牌量接口<br>
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
*
*
* @author yinjihuan
* @date 2016年9月1日
* @param city
@@ -4107,7 +4232,7 @@ public class PropertyController {
/**
* 小区周边一公里均价
*
*
* @author wenyan
* @param model
* @return
@@ -4130,7 +4255,7 @@ public class PropertyController {
/**
* 相同小区相似户型租金范围查询接口
*
*
* @param model
* @return
*/
@@ -4173,7 +4298,7 @@ public class PropertyController {
/**
* 周边小区相似户型租金范围查询接口
*
*
* @param model
* @return
*/
@@ -4236,7 +4361,7 @@ public class PropertyController {
/**
* 室号查询接口
*
*
* @param model
* @return
*/
@@ -4272,7 +4397,7 @@ public class PropertyController {
/**
* 楼栋楼层信息查询接口
*
*
* @param model
* @return
*/
@@ -4297,7 +4422,7 @@ public class PropertyController {
/**
* 楼栋查询接口
*
*
* @param model
* @return
*/
@@ -4342,7 +4467,7 @@ public class PropertyController {
/**
* 楼栋查询接口
*
*
* @param model
* @return
*/
@@ -4795,7 +4920,7 @@ public class PropertyController {
/**
* 房产交易记录查询
*
*
* @param model
* @return
*/
@@ -4855,7 +4980,7 @@ public class PropertyController {
/**
* 房产拍卖记录
*
*
* @param model
* @return
*/
@@ -4916,7 +5041,7 @@ public class PropertyController {
/**
* 房产近期评估次数
*
*
* @param model
* @return
*/
@@ -5412,6 +5537,36 @@ public class PropertyController {
}
}
public Object searchHouseLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber()) || StringUtils.isBlank(model.getRoomNumber())){
mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号,室为必填项"));
return mav;
}
List<HouseLoudong> houseLoudongs = houseLoudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber(), model.getRoomNumber());
mav.setModel(BasicModel.buildOKModel().setResult(houseLoudongs));
return mav;
}
public Object searchLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber())){
mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号为必填项"));
return mav;
}
List<Loudong> loudongs = loudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber());
mav.setModel(BasicModel.buildOKModel().setResult(loudongs));
return mav;
}
public void setHouseLoudongService(HouseLoudongService houseLoudongService) {
this.houseLoudongService = houseLoudongService;
@@ -5601,4 +5756,16 @@ public class PropertyController {
public void setRentDealPriceService(RentDealPriceService rentDealPriceService) {
this.rentDealPriceService = rentDealPriceService;
}
public void setCommunityPoiService(CommunityPoiService communityPoiService) {
this.communityPoiService = communityPoiService;
}
public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) {
this.evaluateFactorService = evaluateFactorService;
}
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
this.communityBasePriceService = communityBasePriceService;
}
}
@@ -0,0 +1,36 @@
package com.fangjia.open.controllers;
import com.fangjia.core.util.AnalyticsUtil;
import com.fangjia.open.interceptors.Authority;
import com.fangjia.open.interceptors.Role;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import javax.servlet.http.HttpServletRequest;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.Map;
public class requestController {
/**
* »ñÈ¡ÇëÇóµÄrequestºÍIPÐÅÏ¢
* @param request
* @return
*/
public Object requestAction(HttpServletRequest request) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
String ip = AnalyticsUtil.getIpAddress(request);
Map<String, String> map = new HashMap<String, String>();
Enumeration headerNames = request.getHeaderNames();
while (headerNames.hasMoreElements()) {
String key = (String) headerNames.nextElement();
String value = request.getHeader(key);
map.put(key, value);
}
map.put("ip", ip);
mav.setModel(BasicModel.buildOKModel().setResult(map));
return mav;
}
}
@@ -99,6 +99,7 @@ public class DistrictInfoDto {
private String salesPhoneNumber;
private Integer buildingCount;
private Integer finishYear;
private String township;
public String getDistrict() {
@@ -818,4 +819,12 @@ public class DistrictInfoDto {
public void setFinishYear(Integer finishYear) {
this.finishYear = finishYear;
}
public String getTownship() {
return township;
}
public void setTownship(String township) {
this.township = township;
}
}
@@ -75,8 +75,8 @@ public class AnalyticsFilter implements Filter {
if ("1".equals(request.getParameter("__refresh_cache"))) {
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
}
createUID(request, response);
createSID(request, response);
//createUID(request, response);
//createSID(request, response);
response.addHeader("Access-Control-Allow-Origin", "*");
response.addHeader("Access-Control-Allow-Methods", "*");
chain.doFilter(request, response);
@@ -8,6 +8,7 @@ public class AuctionModel extends FangjiaApiModel{
private String city;
private String district;
private String address;
private String name;
private Date minStartTime;
private Date maxStartTime;
private Integer minStartPrice;
@@ -17,6 +18,7 @@ public class AuctionModel extends FangjiaApiModel{
private Integer propertyType;
private Integer status;
private Integer limit = 10;
private String startDate;
public String getId() {
return id;
@@ -121,4 +123,20 @@ public class AuctionModel extends FangjiaApiModel{
public void setStatus(Integer status) {
this.status = status;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getStartDate() {
return startDate;
}
public void setStartDate(String startDate) {
this.startDate = startDate;
}
}
@@ -1,6 +1,9 @@
package com.fangjia.open.model;
import java.util.Date;
public class DealPriceModel extends FangjiaApiModel{
private String id;
private String city;
@@ -34,9 +37,13 @@ public class DealPriceModel extends FangjiaApiModel{
private Double size;
private Integer max;
private Integer limit = 10;
private String decoration;
private boolean convert;
private String startDate;
private String endDate;
public String getCity() {
@@ -198,4 +205,36 @@ public class DealPriceModel extends FangjiaApiModel{
public void setConvert(boolean convert) {
this.convert = convert;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getStartDate() {
return startDate;
}
public void setStartDate(String startDate) {
this.startDate = startDate;
}
public String getEndDate() {
return endDate;
}
public void setEndDate(String endDate) {
this.endDate = endDate;
}
public Integer getLimit() {
return limit;
}
public void setLimit(Integer limit) {
this.limit = limit;
}
}
@@ -246,6 +246,42 @@ public class EvaluationModel extends FangjiaApiModel {
private Date buildDate;
private String sid;//微信小程序中的sid
private String code;//微信小程序中的code
private String id;
private String startDate;
private String endDate;
private Integer limit = 10;
public Integer getLimit() {
return limit;
}
public void setLimit(Integer limit) {
this.limit = limit;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getStartDate() {
return startDate;
}
public void setStartDate(String startDate) {
this.startDate = startDate;
}
public String getEndDate() {
return endDate;
}
public void setEndDate(String endDate) {
this.endDate = endDate;
}
public String getRegion() {
return region;
@@ -4,6 +4,7 @@ import com.fangjia.service.datastore.bean.HouseQueryBean;
import com.fangjia.service.query.QueryBean;
public class HouseSearchModel extends SearchModel {
private String id;
private String kind;
private String estateType;
private String buildingType;
@@ -116,7 +117,12 @@ public class HouseSearchModel extends SearchModel {
public void setShowDetail(Boolean showDetail) {
this.showDetail = showDetail;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}
@@ -46,7 +46,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
* Â¥ºÅ
*/
private String unitNumber;
/**
* ÐÂÂ¥ºÅ
*/
private String houseNumber;
private String id;
/**
* ÊÒºÅ
*/
@@ -905,4 +910,20 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
public String getHouseNumber() {
return houseNumber;
}
public void setHouseNumber(String houseNumber) {
this.houseNumber = houseNumber;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
}
@@ -16,6 +16,8 @@ public abstract class SearchModel extends FangjiaApiModel {
protected Integer totalPriceEnd;
protected Integer avgPriceStart;
protected Integer avgPriceEnd;
protected Integer startTime;
protected Integer endTime;
protected Integer areaStart;
protected Integer areaEnd;
protected Integer room;
@@ -358,4 +360,20 @@ public abstract class SearchModel extends FangjiaApiModel {
public void setTotalPrice(Integer totalPrice) {
this.totalPrice = totalPrice;
}
public Integer getStartTime() {
return startTime;
}
public void setStartTime(Integer startTime) {
this.startTime = startTime;
}
public Integer getEndTime() {
return endTime;
}
public void setEndTime(Integer endTime) {
this.endTime = endTime;
}
}
+27
View File
@@ -494,5 +494,32 @@
<artifactId>msgpack</artifactId>
<version>0.6.12</version>
</dependency>
<dependency>
<groupId>com.asprise.ocr</groupId>
<artifactId>java-ocr-api</artifactId>
<version>[15,)</version>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>4.1.0</version>
</dependency>
<dependency>
<groupId>net.sourceforge.tess4j</groupId>
<artifactId>tess4j</artifactId>
<version>3.4.0</version>
<exclusions>
<exclusion>
<groupId>com.sun.jna</groupId>
<artifactId>jna</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
</project>
@@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("city", city)
q.put("status", 1)
// q.put("name","ÁÉÔ´Ò»´å")
def dbcursor = LOU_DONG_STAT.find(q)
def map = [:]
def set = new HashSet()
for(def db : dbcursor){
def list = map.get(db.region+"##"+db.name)
if(!list){
list = []
map.put(db.region+"##"+db.name, list)
}
list << db
if(db.house_num && db.house_num!="--"){
def key = db.region+"##"+db.name +"##"+db.house_num
if(!set.contains(key)){
set << key
list << db
}
}
}
return map
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){
def rets = []
def keys = []
distMap.each{k,dist->
def buildings = buildingMap.get(k)
if(buildings){
@@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{
def ret = []
def areaData = distCodeMap.get(obj.region)
if(areaData){
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << ""
ret << ""
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
def key = dist._id.toString() +"##"+obj.house_num
if(!keys.contains(key)){
keys << key
ret << cityCodeDatas.get(obj.city)
ret << obj._id.toString()
ret << obj.house_num
ret << obj.house_num
ret << ""
}
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
if(arrs){
ret << arrs[0].trim()
}else{
ret << ''
}
ret << areaData.district_code
ret << dist._id.toString()
if(obj.build_year){
ret << obj.build_year
}else{
ret << ""
}
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
if(arrs){
ret << arrs[0].trim()
}else{
ret << ''
}
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
if(obj.floors){
ret << obj.floors
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1){
// ret << "1"
// }else{
// ret << "0"
// }
ret << ""
ret << ""
// if(obj.elevators && obj.elevators>=1) {
// ret << obj.elevators//µçÌÝÊý
// }else{
// ret << ""
// }
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << "1"
rets << ret
}
}
}
}
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{
}else{
def jsonObj = null
if(dist.address){
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
}
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
if(jsonObj && jsonObj.street){
if(jsonObj.street){
ret << jsonObj.street
}else{
ret << ""
}
if(jsonObj.streetNumber){
ret << jsonObj.streetNumber
}else{
ret << ""
}
}else{
ret << ""
ret << ""
@@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
q.put("status", 0)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
@@ -408,7 +408,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -449,6 +449,13 @@ class BigdataForsaleBatch extends BaseBatch{
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="ÆÕͨסլ"){
if(seawater.estate_type&& seawater.estate_type.contains("ÉÌס")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
}
if(!seawater.total_price){
continue;
}
@@ -367,7 +367,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","特房锦绣一方")
// q.put("name","辽源一村")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["ĆŐͨץՏ","ÉĚץ"]))
@@ -57,6 +57,12 @@ class CcbFtpUtils {
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
}
static def listFileNames(month){
//上传到ftp
sftpChannel = reConnect();
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
return files
}
static main(args) {
@@ -15,8 +15,8 @@ class DealPriceDataProcess {
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
// def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
@@ -25,8 +25,8 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()
task.delDealByPe()
task.deleteThirdPartyData()
// task.delDealByPe()
}
def delDealByPe(){
@@ -370,26 +370,27 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = ["¶«Ý¸"] //,"º¼ÖÝ""¹ãÖÝ",
def citys = CcbCommonUtils.citys
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def publishTime = '2021-10-01'
def month = 202110
def month = 202204
def rows = []
for(def city : citys) {
def rets = findPeGteThirdParty(city, 202111)
def rets = findPeGteThirdParty(city, 202205)
def total = 0
println city +" "+rets.size()
for(def m : rets){
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
sqlInstance.executeUpdate(sql)
sqlInstance236.executeUpdate(sql)
}
println city +" "+total
}
}
def findPeGteThirdParty(cityName, yearmonth){
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='µÚÈý·½' and cityname='${cityName}' and pe>20"
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='µÚÈý·½' and cityname='${cityName}' and pe>50"
def rets = []
sqlInstance2.eachRow(sql,{r->
sqlInstance236.eachRow(sql,{r->
def map = [:]
map.commid = r.commid
map.price = r.target_avg_price
@@ -38,8 +38,8 @@ class ExportOracleData {
exportCommunityBuileType()
exportThirdPartyPrice2(lastMonth)
exportBigdataForsale("2022-02-19")
exportBigdataDeal("2022-02-01")
exportBigdataForsale("2022-04-19")
exportBigdataDeal("2022-04-01")
//计算结果相关
@@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
import org.apache.commons.codec.digest.DigestUtils
class ExportOracleEvaluationData {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
@@ -33,6 +35,8 @@ class ExportOracleEvaluationData {
exportCommCaseDetailGj(curMonth)
def curDate = curMonth.toString()+"28"
// 生成MD5文件
exportMd5()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// okÎļþ
@@ -413,4 +417,23 @@ class ExportOracleEvaluationData {
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
}
def exportMd5(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
Vector files = CcbFtpUtils.listFileNames(curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
}
}
def fileName = "/ccb/md5.txt"
if(CommonDatautils.isWindow()){
fileName = "D:\\房价网\\md5.txt"
}
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "md5.txt 上传Ftp完成......"
}
}
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1")
def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
@@ -1,21 +1,29 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import groovy.sql.Sql
import org.bson.types.ObjectId
class LoadGuide {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new LoadGuide()
task.run()
}
def run(){
addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
@@ -26,9 +34,51 @@ class LoadGuide {
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
}
def addPersonGuided(filename){
ExcelUtils.modify(filename, { d ->
def city = d['B']
if(city){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
createSql( d, cityCodeDatas, distCodeMap)
}
})
}
def createSql(d, cityCodeDatas, distCodeMap){
def id = new ObjectId()
def commid = d['A']
def cityId = cityCodeDatas.get(d['B'])
def sampleRegion = d['C'].replace("街道","").replace("镇","")
def distRegion = distCodeMap.get(sampleRegion)
def cityName = d['B']
def distName = d['C']
def commName = d['D']
def price = d['E'].toDouble().toInteger()
def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district")
if(dist){
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
def dealDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " +
"('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')"
println sql
sqlInstance.executeUpdate(sql)
}else{
println " 没查到 "+commName
}
}
def match(def filename){
def city = "西安"
ExcelUtils.modify(filename,{d->
@@ -0,0 +1,67 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DistrictCodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* oracle中小区统计数据同步至85中
*/
class OracleDistrict2Mongo85 {
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017)
static main(args) {
// 只写入重庆市数据
def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ"
def count = 0
sqlInstance.eachRow(sql, { r ->
def city = DistrictCodeUtils.getCityName(r['cityid'])
def region = RegionData.formatRegion(city, r['distname'])
DBObject dbObject = new BasicDBObject("_id", r['commid'])
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
.append("name", r['commname'])
.append('subway', Integer.parseInt(r['subway'].toString()))
.append("traffic", Integer.parseInt(r['traffic'].toString()))
.append("bank", Integer.parseInt(r['bank'].toString()))
.append("primary", Integer.parseInt(r['primary'].toString()))
.append("junior", Integer.parseInt(r['junior'].toString()))
.append("hospital", Integer.parseInt(r['hospital'].toString()))
.append("funeral", Integer.parseInt(r['funeral'].toString()))
.append("c_date", new Date());
communityPoiConn.update(dbObject, updateObject, true, false)
count++;
if (count % 500 == 0) {
println "已写入" + count + "条小区数据"
}
});
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DistrictCodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
/**
* oracle中小区置信度数据同步至85中
*/
class OracleDistrictBasePrice2Mongo85 {
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017)
static main(args) {
def sql = "SELECT cityname, distname, commid, commname, confidence FROM community_base_price_gj WHERE YEAR_MONTH='202205'"
def count = 0
sqlInstance.eachRow(sql, { r ->
def city = r['cityname']
def region = RegionData.formatRegion(city, r['distname'])
DBObject dbObject = new BasicDBObject("_id", r['commid'])
if (r['confidence'] != null) {
def credit = new BigDecimal(r['confidence']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble()
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
.append("name", r['commname'])
.append("credit", credit)
.append("c_date", new Date());
communityBasePriceConn.update(dbObject, updateObject, true, false)
count++;
} else {
print "置信度为空:" + r.toString()
}
if (count % 500 == 0) {
println "已写入" + count + "条小区数据"
}
});
}
}
@@ -225,17 +225,42 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01")
//def testMap = findCaluateCityPPE(sqlInstance236, 202203)
//ExcelUtils.modify(fileName,{d->
// if(d['C']){
// def oldcount = dealMap.get(d['C'])
// d['V'] = oldcount
// d['X'] = oldcount
// def casecount = testMap.get(d['C'])
// d['W'] = casecount
// d['Y'] = casecount
// println d
// }
//},0)
def rows = findTradePpeData(sqlInstance236)
// rows.unique()
def map=findTradeDetail(sqlInstance236,null)
def datas=[]
map.each{key,row->
if(!rows.contains(key)){
datas<<row
}
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def datas=[]
//cityCodeDataMap.each{city,code->
// def rows = findTradePpeData(sqlInstance236, 202203,code)
// def alltrades=findTradeDetail(sqlInstance236,code)
// alltrades.each{it->
// def commid=it[0]
// if(!rows.contains(commid)){
// datas<<it
// }
// }
// println datas.size()
//}
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
// if(d['I'] && d['K']){
@@ -270,24 +295,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
// }
//},0)
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
def flag = d['M']
if(flag==0.0){
if(d['D']){
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(d['D']))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
index++
println index
}
}
},0)
//def index = 0
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
// def flag = d['M']
// if(flag==0.0){
// if(d['D']){
// def udata = new BasicDBObject()
// udata.put("name_cat", 11)
// udata.put("m_date",new Date())
//
// def q = new BasicDBObject()
// q.put("_id", new ObjectId(d['D']))
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// index++
// println index
// }
// }
//
//},0)
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
// if(d['A']){
@@ -558,8 +583,30 @@ def findCaluateCityPPE(def sqlInstance, def yearMonth){
}
def findTradePpeData(def sqlInstance){
def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'";
def rows = new HashSet()
sqlInstance.eachRow(sql,{r->
rows<<r.commid+"##"+r.buildarea+"##"+r.target_avg_price+"##"+r.dealdate
});
return rows
}
def findTradeDetail(def sqlInstance, def cityid){
def sql = "select commid,cityname,distname,commname,buildarea,dealunitprice,to_char(dealdate,'yyyy-mm-dd') as dealdate from bigdata_deal_gj where dealdate>=to_date('2022-02-01','yyyy-MM-dd')";
def map = [:]
sqlInstance.eachRow(sql,{r->
def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate
def row=[]
row<<r.commid
row<<r.cityname
row<<r.distname
row<<r.commname
row<<r.buildarea
row<<r.dealunitprice
map.put(key,row)
});
return map
}
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
def count = 0
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= 202202//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202112
def endDate = 202202
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def startDate = 202202
def endDate = 202204
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -40,8 +40,8 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
task.run()//统计最近3个月抓取的成交数据量
// task.exportTrade()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -75,13 +75,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2021-12-01"
def date = "2022-01-01"
def endDate = "2022-02-01"
def newDate = "2022-03-01"
def startDate = "2022-02-01"
def date = "2022-03-01"
def endDate = "2022-04-01"
def newDate = "2022-05-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -131,7 +131,7 @@ class TradeStatLog {
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
@@ -481,8 +481,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2022-02-01"
def endDate = "2022-03-01"
def startDate = "2022-03-01"
def endDate = "2022-04-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -0,0 +1,155 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
class CcbDataStat {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new CcbDataStat()
// task.run()
task.tradeRun()
}
def tradeRun(){
def ccbCitys = CcbCommonUtils.citys;
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d ->
if(d['A']){
if(ccbCitys.contains(d['A'])){
d['D'] = findOracleTotalCount(d['A'])
}else{
def totalcount = findAllTradeCount(d['A'], startDate, endDate)
def repeatCount = findRepeatCount(d['A'], startDate, endDate)
def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate)
def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate)
d['B'] = totalcount
d['C'] = repeatCount
d['D'] = matchTotalCount
d['E'] = matchRepeatTotalCount
}
}
})
}
def findRepeatCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcursor = DEAL_PRICE_COLL.find(q)
def set = new HashSet()
for(def d : dbcursor){
def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
if(!set.contains(key)){
set << key
}
}
return set.size()
}
def findMatchRepeatTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def dbcursor = DEAL_PRICE_COLL.find(q)
def set = new HashSet()
for(def d : dbcursor){
def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
if(!set.contains(key)){
set << key
}
}
return set.size()
}
def findMatchTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
public static def findAllTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
// q.put("match_id", new BasicDBObject('$exists', false))
def total = DEAL_PRICE_COLL.count(q)
return total
}
def findOracleTotalCount(city){
def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate<to_date('2022-01-01','yyyy-MM-dd')";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def run(){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
if(d['A']) {
def citycode = cityCodeDataMap.get(d['A'])
def total = findTotalCount(citycode)
def f_count = findDistFinishDate(citycode)
def b_count = findBuildTypeCount(citycode)
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
d['C'] = f_pre
d['D'] = b_pre
d['E'] = total
println d['A'] +" "+ total +" "+f_count+" "+b_count
}
})
}
def findTotalCount(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findDistFinishDate(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findBuildTypeCount(citycode){
def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
}
@@ -0,0 +1,117 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
class CcbDistNameMatch {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new CcbDistNameMatch();
// task.run()
task.run1()
}
def run1(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
if(d['D'].contains("、")){
println d['D']
}
def arrs = d['D'].split("、")
def list = []
def flags = []
for(def s : arrs){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district")
if(dist){
list << dist.name
flags << "完全一致"
}else{
def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion)
if(jsonObject != null && jsonObject.precise!=-1) {
if(jsonObject.get("district") == newRegion){
list << jsonObject.get("name")
flags << "匹配接口"
}
}
}
}
if(list){
list = list.unique()
d['E'] = list.join(",")
d['F'] = flags.join(",")
}
}
}
})
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d ->
def regionCode = null
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
d['D'] = dist.name
def count = findDistrictByOracle(dist._id.toString())
if(count && count>0){
d['F'] = 1
}else{
d['H'] = dist.estate_type2
d['J'] = dist.name_cat
}
}else{
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
if (rets) {
if(rets.size()>1){
d['E'] = "别名-多个"
}else{
dist = rets[0]
d['E'] = d['C'] +"--"+dist.name
def count = findAliasByOracle(dist._id.toString())
if(count && count>0) {
d['G'] = 1
}else{
d['H'] = dist.estate_type2
d['J'] = dist.name_cat
}
}
}
}
}
}
});
}
def findDistrictByOracle(commid){
def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findAliasByOracle(commid){
def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
}
@@ -2,25 +2,248 @@ package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.mongodb.BasicDBObject
import java.util.concurrent.CopyOnWriteArrayList
class CcbDistrictToDB {
static main(args) {
def task = new CcbDistrictToDB();
// task.run()
task.run1()
// task.run1()
// task.meituanRun()
task.meituanSqrentRun()
// task.updateDistrict()
// task.updateDongguangRegion()
}
def updateDongguangRegion(){
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"])
for(def d : dist){
def t = d.c_date
if(t.getTime()>cDate.getTime()){
}
}
}
def updateDistrict(){
def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"]
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
def index = 0
for(def city : citys){
def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"])
for(def d : dist){
def t = d.c_date
if(t.getTime()>cDate.getTime()){
if(d.name.contains("(")){
def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1)
def newName = d.name.replace(delstr,"")
index++
println index+" "+d.city+" "+d.name+" "+newName
def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district")
if(!obj){
println "update"
def udata = new BasicDBObject()
udata.put("name", newName)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
d.name = newName
addPrice(d)
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0)
}else{
println "delete"
DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
}
}
}
}
}
}
def meituanSqrentRun(){
def index = 0
ExcelUtils.modify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
index++
println index+" "+d['H']
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion) {
def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name)
if(name){
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
}
}
},0);
}
def meituanRun(){
def index = 0//美团需入库小区合并后.xlsx
def keys = new CopyOnWriteArrayList()
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
index++
println index+" "+d['H']
def city = d['F']
def region = d['G']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion) {
def name = NameProcess.nameStandard(city, newRegion, d['H'])
name = nameFileter(name)
if (name) {
def formatAddress = null
if (d['I']) {
formatAddress = AddressUtil.formatAddress(city, d['I'])
}
// if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
// def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
// def map = findStandardObject2(city,region, name, d['E'], rets);
// if(map && map.address){
// formatAddress = AddressUtil.formatAddress(city, map.address)
// if(!formatAddress){
// if(map.address.contains("交叉口")){
// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
// }else if(map.address.contains("交叉")){
// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
// }
// }
// }
// }
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if (d['L']) {
glat = d['M'].toString().toDouble() * 10000000000L
glng = d['L'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if (name.contains("别墅")) {
estateType2 = "别墅"
} else if (name.contains("商住")) {
estateType2 = "商住"
} else {
def type = d['J']
if (type) {
if (type.contains("别墅")) {
estateType2 = "别墅"
} else if (type.contains("商住")) {
estateType2 = "商住"
}
}
}
def key = city + "##" + newRegion + "##" + name
if (name && !keys.contains(key)) {
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if (!dist) {
keys << key
def data = new BasicDBObject();
data.put("name_cat", 12)
data.put("status", 0)
data.put("cat", "district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region", newRegion)
data.put("name", name)
data.put("estate_type2", estateType2)
data.put("address", formatAddress)
if (glat) {
data.put("lat", glat)
data.put("lng", glng)
data.put("lat2", blat)
data.put("lng2", blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map", markMap)
index++
DbUtils.seaweed.save(data);
println index + " " + data
def stop = false
while (!stop) {
if (data._id.toString()) {
addPrice(data)
stop = true;
}
}
}
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
},0);
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def run1(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
def city = d['A']
def region = d['B']
def newRegion = RegionData.formatRegion(city, region)
@@ -71,6 +294,7 @@ class CcbDistrictToDB {
if(name){
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
@@ -216,9 +440,85 @@ class CcbDistrictToDB {
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
}
return name;
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -1,10 +1,12 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import com.fangjia.core.util.BaiDuApiUtils
@@ -13,7 +15,82 @@ class DistrictMarge {
static main(args) {
def task = new DistrictMarge()
//task.run()
task.repeatAddress()
// task.repeatAddress()
// task.removeDist()
task.updateAddress()
// task.addMappingAlias()
}
def updateAddress(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d ->
if (d['A']) {
def region = d['B'].trim()
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
DBObject udata = new BasicDBObject()
udata.put("address", d['D'])
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
});
}
def removeDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
DBObject udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
})
}
def addMappingAlias(){
def index = 0
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
index ++
if (d['A']) {
def region = d['B'].trim()
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if (dist) {
def mapping_alias = dist.mapping_alias
if(!dist.mapping_alias){
mapping_alias = []
}
def alias = d['D']
def arrs = alias.split("、")
for(def s : arrs){
if(!mapping_alias.contains(s)){
mapping_alias << s
}
}
DBObject udata = new BasicDBObject()
udata.put("mapping_alias", mapping_alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
}
}else{
println d['A'] + " 格式化失败 "+d['B']
}
}
})
}
def repeatAddress(){
@@ -36,10 +36,37 @@ class MatchMapData {
// task.run3()
// task.exportHouse()
// task.addDistrictLevel()
task.run4()
// task.run4()
task.run5()
}
def run5(){
ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\Êý¾ÝºËʵ.xlsx", { d ->
if (d['A']) {
def region = d['B']
def newRegion = RegionData.formatRegion(d['A'], region)
if (newRegion) {
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
if(dist){
if(dist.mapping_alias){
d['D'] = dist.mapping_alias.join(",")
}
d['E'] = dist.address
}
def dist1 = CommonDatautils.findDistrict(d['A'], newRegion, d['G'], "district")
if(dist1){
if(dist1.mapping_alias){
d['H'] = dist1.mapping_alias.join(",")
}
d['I'] = dist1.address
}
}
}
});
}
def run4() {
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
def regionCode = null
@@ -0,0 +1,108 @@
package com.fangjia.script.ccbevaluation.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class OracleHouse {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new OracleHouse()
task.run()
}
def run(){
def rets = findAllHouse("2022-04-19")
def totalCount = new AtomicLong()
println rets.size()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
def arrs = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def flag = findHouseFlag(arrs[0])
println totalCount.get()+" "+ arrs[0] +" "+flag
if(flag){
def dist = CommonDatautils.findDistrictById(arrs[1])
if(dist && dist.estate_type2=="ÆÕͨסլ"){
removeHouse(arrs[0])
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
}
def findHouseFlag(def id){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_comp_potential", queryBuilder);
def delFlag = false
for(def h : houses){
if(h.estate_type && (h.estate_type.contains("ÉÌ") || h.estate_type.contains("°ì"))){
println id+" "+h.estate_type
delFlag = true
}
}
return delFlag
}
def findAllHouse(def publishDate) {
def sql = "select FORSALEID,COMMID from bigdata_forsale_gj where publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
def allRows = []
def index = 0
sqlInstance.eachRow(sql, { r ->
index = index + 1
println index
def data = []
data << r.FORSALEID
data << r.COMMID
allRows << data
});
return allRows
}
def removeHouse(houseid){
def sql = "delete from bigdata_forsale_gj where FORSALEID='${houseid}' ";
sqlInstance.executeUpdate(sql)
}
}
@@ -0,0 +1,277 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
/**
* 重庆估值测试--2小时
*/
class EvaluationBatch {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new EvaluationBatch()
// task.run()
// task.run1();
task.run2()
}
def run2(){
def startDate = DateUtils.parseFromTightYYYYMMDD("2021-01-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-07")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V1.xlsx", { row ->
if(row['N'] && row['N'].toString().toDouble()>=20){
println row['F'] +" "+row['N']
def dist = CommonDatautils.findDistrictById(row['C'])
for(def weekend : weekends){
CommonUtils.recalculationAverage(row['D'], row['E'], row['F'], dist.estate_type2, weekend,false,0)
}
// FangjiaClientUtils.sendDistrictPriceCalculate(row['D'], row['E'], row['F'], 48);
}
},0)
}
def run1(){
def city = "重庆"
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V2.xlsx", { row ->
def region = row['E']
def name = row['F']
def id = row['C']
def lastWeekend = DateUtils.getFirstDayOfWeek(new Date()-30)
def lastYearWeekend = DateUtils.getFirstDayOfWeek(new Date()-365)
if (region && name) {
row['H'] = CommonDatautils.findNewSqmetreRent(city, region, name)
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
if(priceObj){
row['AB'] = priceObj.avg_price
}
def dist = CommonDatautils.findDistrictById(id)
if(dist) {
List<DBObject> list = DbUtils.findAroundDistrict(dist)
def map = getArroundAvgPriceMap(list)
if(map){
row['L'] = map.avg_price
row['Q'] = map.max_val
row['R'] = map.min_val
}
def lastArroundPrice = getArroundAvgPriceByWeekend(list, lastWeekend)
if(lastArroundPrice && priceObj){
def mod = (map.avg_price-lastArroundPrice)*100/lastArroundPrice
mod = DataHandleUtils.getDoubleRound(2, mod)
row['P'] = mod
}
def lastYearArroundPrice = getArroundAvgPriceByWeekend(list, lastYearWeekend)
if(lastYearArroundPrice && priceObj){
def mod = (map.avg_price-lastYearArroundPrice)*100/lastYearArroundPrice
mod = DataHandleUtils.getDoubleRound(2, mod)
row['O'] = mod
}
}
def lastWeekendObj = CommonDatautils.queryDistrictPrice(city, region, name, lastWeekend)
if(lastWeekendObj){
row['S'] = lastWeekendObj.avg_price
if(priceObj){
def mod = (priceObj.avg_price-lastWeekendObj.avg_price)*100/lastWeekendObj.avg_price
mod = DataHandleUtils.getDoubleRound(2, mod)
row['M'] = mod
}
}
def lastYearObj = CommonDatautils.queryDistrictPrice(city, region, name, lastYearWeekend)
if(lastYearObj){
row['T'] = lastYearObj.avg_price
if(priceObj){
def mod = (priceObj.avg_price-lastYearObj.avg_price)*100/lastYearObj.avg_price
mod = DataHandleUtils.getDoubleRound(2, mod)
row['N'] = mod
}
}
}
},0)
}
def run(){
def city = "重庆"
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆估值交易信息V2.xlsx", { row ->
def region = row['E']
def name = row['F']
def id = row['C']
def area = row['G'].toDouble()
if(region && name){
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
if(priceObj){
row['J'] = priceObj.avg_price
if(row['C']){
row['I'] = (area*priceObj.avg_price).toInteger()
}
}
def dist = CommonDatautils.findDistrictById(id)
if(dist){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
row['N'] = getArroundAvgPrice(list)
def alias = dist.alias
if(!alias){
alias = []
}
alias << dist.name
def houseAvgPrice = findSimAreaHousePrice(dist.city, dist.region, alias,20220101,20220401, area-10, area+10)
if(houseAvgPrice){
row['K'] = houseAvgPrice
}
def tradeAvgPrice = findTrades(dist, DateUtils.parseFromTightYYYYMMDD("2021-12-01"), area-10, area+10)
row['L'] = tradeAvgPrice
def avg_price = findAvgPrice(dist._id.toString())
if(row['J'] && avg_price){
def weekendPrice = row['J'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['P'] = ppe15
}
}
}
});
}
def findTrades(dist, startDate, startArea, endArea){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
// q.put("source", "链家在线")
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate))
q.put("area", new BasicDBObject('$gte', startArea).append('$lt',endArea))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def prices = []
for(def d : dbcorsor){
if(d.avg_price){
prices << d.avg_price
}
}
def price = null
if(prices){
price = (prices.sum()/prices.size()).toInteger()
}
return price
}
def getArroundAvgPriceMap(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
prices.sort()
def map = [:]
map.min_val = prices[0]
map.max_val = prices[prices.size()-1]
def avg_price = (prices.sum()/prices.size()).toInteger()
map.avg_price = avg_price
return map;
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
def getArroundAvgPriceByWeekend(def list, def weekend){
def prices = []
for(def l : list){
def priceObj =CommonDatautils.queryDistrictPrice(l.city, l.region, l.name, weekend)
if(priceObj){
prices << priceObj.avg_price
}
}
def avg_price = null;
if(prices){
avg_price = (prices.sum()/prices.size()).toInteger()
}
return avg_price;
}
def findSimAreaHousePrice(def city, def region, def alias, def startDate, def endDate, def startArea, def endArea){
def aliasStr = null;
def total = 0
def prices = []
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and area>=${startArea} and area<=${endArea}"
ElasticUtils.executeSqlQuery(sql,{ data->
prices << data.avg_price
})
}catch(Exception e){
e.printStackTrace()
}
def price = null
if(prices){
price = (prices.sum()/prices.size()).toInteger()
}
return price
}
public String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -0,0 +1,84 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class BlankMatch {
static main(args) {
def task = new BlankMatch()
task.run()
// task.run1()
// task.run2()
// task.run3()
}
def run() {
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
//def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx";
def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['C']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['C'] = jsonObject.get("id")
row['D'] = jsonObject.get("city")
row['E'] = jsonObject.get("district")
row['F'] = jsonObject.get("name")
row['G'] = jsonObject.get("address")
}
}
});
}
def run1() {
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx", { row ->
if(row['F']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['F'], "property", false, row['E'], null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['C'] = jsonObject.get("id")
row['D'] = jsonObject.get("city")
row['J'] = jsonObject.get("name")
}
}
// def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
// if(jsonObject != null && jsonObject.precise!=-1) {
// row['C'] = jsonObject.get("id")
// row['D'] = jsonObject.get("city")
// row['E'] = jsonObject.get("district")
// row['F'] = jsonObject.get("name")
// row['G'] = jsonObject.get("address")
// }
});
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
if(dist){
row['C'] = dist._id.toString()
}
}
},0)
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
if(!row['C']){
for(def d : datas){
if(d['B'] == row['B']){
row['C'] = d['C']
row['D'] = d['D']
row['E'] = d['E']
row['F'] = d['J']
break;
}
}
}
},0)
}
}
@@ -0,0 +1,77 @@
package com.fangjia.script.chongqing
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import org.apache.lucene.queryparser.classic.QueryParser
import org.bson.types.ObjectId
class FieldAddValue {
static main(args) {
def task = new FieldAddValue()
task.run()
}
def run(){
//16991 14360
def districts = CommonDatautils.findAllDistrictByCity("ÖØÇì", "district", ["region","name","alias","is_elevator"])
def index = 0
def updateIndex = 0
for(def d : districts){
if(!d.is_elevator){
index++
println index
def alias = d.alias
if(!d.alias){
alias = []
}
alias << d.name
def maxHeight = findHouseHeight("ÖØÇì", d.region, alias, 20220101)
if(maxHeight && maxHeight>6){
updateIndex++
def udata = new BasicDBObject()
udata.put("is_elevator", true)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId( d._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
println index+" "+updateIndex
}
def findHouseHeight(def city, def region, def alias, def startDate){
def aliasStr = null;
def total = 0
def heights = null
try{
aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select max(height) as height from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name in ('${aliasStr}') "
ElasticUtils.executeSqlQuery(sql,{ data->
if(data.height){
heights = data.height
}
})
}catch(Exception e){
e.printStackTrace()
}
// def max_height = null
// if(heights){
// heights.sort()
// max_height = heights[heights.size()-1]
// }
return heights
}
public String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
}
@@ -0,0 +1,612 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.zip.GZIPInputStream
class MatchEvaluateRegionTest {
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\美团\\10城小区明细-V10.xlsx"
def total = 1500
static main(args) {
def task = new MatchEvaluateRegionTest()
// task.run()//小区匹配
// task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
// task.addPrice()
// task.addArroundPrice()
// task.addArroundPriceByLat()
// task.addRentRate()
// task.addAuthor()
// task.addLevel()
// task.addSorce()
// task.addArroundScore()
task.pkPrice()
}
def run(){
def index = 0
ExcelUtils.multiThreadingModify(path, { row ->
index ++
if(!row['F']){
println index+" "+row['E']
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false,row['B'],null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("name")
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
row['M'] = jsonObject.get("lng")
row['N'] = jsonObject.get("lat")
}
}
});
}
def run1(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F'] && row['A']){
def jsonObject = OpenApiClient.addressMatch(row['A'], row['E'], null, true, row['B'],row['C'].toString())
if(jsonObject != null && jsonObject.precise!=-1) {
row['F'] = jsonObject.get("id")
row['G'] = jsonObject.get("city")
row['H'] = jsonObject.get("district")
row['I'] = jsonObject.get("name")
row['J'] = jsonObject.get("address")
row['K'] = jsonObject.get("propertyType")
row['L'] = jsonObject.get("map")
row['M'] = jsonObject.get("lng")
row['N'] = jsonObject.get("lat")
}
}
});
}
def pkPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['R'] && row['R'].toString().toDouble().toInteger()==1){
def obj = CommonDatautils.queryDistrictPrice(row['G'], row['H'], row['I'], DateUtils.parseFromTightByPattern("2022-05-22","yyyy-MM-dd"))
if(!obj){
row['R'] = ''
row['S'] = "无最新价"
}
}
});
}
def addPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['L']){
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
if(priceObj){
row['O'] = priceObj.avg_price
// row['N']= (int)(priceObj.avg_price * row['E'])
}
def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['G'], row['H'], row['I'])
if(sqmetreRentPrice){
row['P'] = sqmetreRentPrice
}
// def avg_price = findAvgPrice(row['F'])
// if(row['M'] && avg_price){
// def weekendPrice = row['M'].toString().toDouble()
// def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
// row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
// }
}
});
}
def addRentRate(){
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
def radioMap = [:]
for(def city : citys) {
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
radioMap.put(city, avg_radio)
}
ExcelUtils.multiThreadingModify(path, { row ->
def city = row['G']
if (row['F'] && row['O'] && !row['P']) {
def avg_rent_radio = radioMap.get(city)
if(avg_rent_radio){
row['P'] = (int)(row['O']/avg_rent_radio)
row['S'] = "租金推算1"
}
}
})
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['O']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['E'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['O'] = priceObj.unitPrice
// row['N'] = (int) (priceObj.unitPrice * row['E'])
row['S'] = "周边价格"
}
}
});
}
def addAuthor(){
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['N'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def addDistrict(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def flag = d['M']
if(!flag){
def city = d['E']
def region = d['F']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['G']
name = nameFileter(name)
def formatAddress = null
def address = d['H']
if(address){
formatAddress = AddressUtil.formatAddress(city, address)
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
addPrice(data)
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
}
},0)
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name.contains("(")){
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
name = name.replace(delstr,"")
}
return name;
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
def addSorce(){
ExcelUtils.multiThreadingModify(path, { d ->
if(d['F']){
def score = findDistrictScore(d['F'])
if(score){
d['R'] = score
}
}
});
}
def addArroundScore(){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['R'] && d['F']){
def dist = null
if(d['F'].toString().size()>16){
dist = CommonDatautils.findDistrictById(d['F'])
}
if(!dist){
dist = new BasicDBObject()
dist.city = d['G']
dist.lng = d['M'].toString().toDouble()*10000000000L
dist.lat = d['N'].toString().toDouble()*10000000000L
}
if(dist.lat){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
if (list) {
list = sortByDistance(dist, list);
if(list && list.size()>10){
list = list[0..9]
}
if(list){
def scores = []
for(def s : list){
def score = findDistrictScore(s._id.toString())
if(score){
scores << score
}
}
def arround_score =(int)(scores.sum()/scores.size())
d['R'] = arround_score
d['T'] = "周边小区评分2"
}
}
}
}
});
}
def addArroundPriceByLat(){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['O'] && d['F'] && (d['K'].contains("住宅") || d['K'].contains("商住") ||d['K'].contains("别墅"))){
def dist = null
if(d['F'].toString().size()>16){
dist = CommonDatautils.findDistrictById(d['F'])
}
if(!dist){
dist = new BasicDBObject()
dist.city = d['G']
dist.lng = d['M'].toString().toDouble()*10000000000L
dist.lat = d['N'].toString().toDouble()*10000000000L
}
if(dist.lat){
List<DBObject> list = DbUtils.findAroundDistrict(dist)
if (list) {
list = sortByDistance(dist, list);
if(list && list.size()>10){
list = list[0..9]
}
if(list){
def prices = []
for(def s : list){
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def arround_price =(int)(prices.sum()/prices.size())
d['O'] = arround_price
d['S'] = "周边均价3"
}
}
}
}
}
});
}
def sortByDistance(d, list){
def newList = []
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
newList << nb
}
//排序
Collections.sort(newList, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return newList;
}
def findDistrictScore(id){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
def indexObj = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
def score = 0
for(def h : indexObj){
score = h.total_price_rate
if(score){
break;
}
}
return score
}
def addLevel() {
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
ExcelUtils.multiThreadingModify(path, { row ->
def address = row['E']
def city = RegionData.formatCity(row['A'])
if(row['K'] && !row['K'].contains("住宅") && !row['K'].contains("商住") && !row['K'].contains("别墅")){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
}
}
}
})
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
return datas;
}
def getArroundDistrictPriceByAddress(def city, def address, def levels){
def json = GaoDeApiUtils.geo(city, address)
def location = null
def detail = new BasicDBObject()
def level = null
def rank = null
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
location = m.location
level = m.level
}
}
if(location){
def arrs = location.split(",")
detail.lng = arrs[0].toString().toDouble()*10000000000L
detail.lat = arrs[1].toString().toDouble()*10000000000L
}
if(detail.lng){
detail.city = city
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
if(list && list.size()>10){
list = sortByDistance(detail, list);
try{
list = list[0..9]
}catch(Exception e){
println list.size()
}
}
def arroundAvgPrice = getArroundAvgPrice(list)
if(arroundAvgPrice){
def mod = Integer.MAX_VALUE;
for(def d : levels){
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
if(newMod<mod){
mod = newMod
rank = d.level
}
}
}
}
def m = [:]
m.level = level
m.rank = rank
return m
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
return null
}
}
@@ -0,0 +1,280 @@
package com.fangjia.script.chongqing
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import com.fangjia.script.gj.district.FangjiaClientUtils
class MatchEvaluateTest {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx"
def total = 1500
static main(args) {
def task = new MatchEvaluateTest()
task.run()//小区匹配
task.run1()//地图匹配
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
task.addPrice()
task.addArroundPrice()
task.addAuthor()
// task.stat()
}
def addArroundPrice(){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['A'] && !row['K']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['K'] = priceObj.unitPrice
row['L'] = (int) (priceObj.unitPrice * row['C'])
row['O'] = "周边价格"
}
}
});
}
def run(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("address")
row['I'] = jsonObject.get("propertyType")
row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def run1(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['D'] && row['B']){
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true)
if(jsonObject != null && jsonObject.precise!=-1) {
row['D'] = jsonObject.get("id")
row['E'] = jsonObject.get("city")
row['F'] = jsonObject.get("district")
row['G'] = jsonObject.get("name")
row['H'] = jsonObject.get("address")
row['I'] = jsonObject.get("propertyType")
row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
}
}
});
}
def addPrice(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['J'] && row['E']){
def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G'])
if(priceObj){
row['K'] = priceObj.avg_price
row['L']= (int)(priceObj.avg_price * row['C'])
}
def avg_price = findAvgPrice(row['D'])
if(row['K'] && avg_price){
def weekendPrice = row['K'].toString().toDouble()
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def addAuthor(){
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['N'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def addDistrict(){
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
def keys = new CopyOnWriteArrayList()
def index = 0
ExcelUtils.modify(path, { d->
def flag = d['M']
if(!flag){
def city = d['E']
def region = d['F']
def newRegion = RegionData.formatRegion(city, region)
if(newRegion){
def name = d['G']
name = nameFileter(name)
def formatAddress = null
def address = d['H']
if(address){
formatAddress = AddressUtil.formatAddress(city, address)
if(!formatAddress){
if(address.contains("交叉口")){
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
}else if(address.contains("交叉")){
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
}
}
}
Long glat = null
Long glng = null
Long blat = null
Long blng = null
if(d['K']){
glat = d['L'].toString().toDouble() * 10000000000L
glng = d['K'].toString().toDouble() * 10000000000L
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
blng = baidu.x * 10000000000L
blat = baidu.y * 10000000000L
}
def estateType2 = "普通住宅"
if(name.contains("别墅")){
estateType2 = "别墅"
}else if(name.contains("商住")){
estateType2 = "商住"
}else{
def type = d['I']
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
}else if(type.contains("商住")){
estateType2 = "商住"
}
}
}
def key = city+"##"+newRegion+"##"+name
if(name && !keys.contains(key)){
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
if(!dist){
keys << key
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",newRegion)
data.put("name", name)
data.put("estate_type2",estateType2)
data.put("address", formatAddress)
if(glat){
data.put("lat",glat)
data.put("lng",glng)
data.put("lat2",blat)
data.put("lng2",blng)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
index++
DbUtils.seaweed.save(data);
println index+" "+data
addPrice(data)
}
}
}else{
println city +" "+region +" 区域无法标准化"
}
}
},0)
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
if(name.contains("(")){
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
name = name.replace(delstr,"")
}
return name;
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
def stat(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1)
ExcelUtils.multiThreadingModify(path, { d ->
if(d['B']){
for(def row : datas){
if(d['B']==row['B']){
if(row['K']){
d['P'] = row['K']
break
}
}
}
}
});
}
}
@@ -0,0 +1,666 @@
package com.fangjia.script.chongqing
import com.fangjia.core.util.SourceNameUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
class PocTest {
def total = 3259
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new PocTest();
task.run();
}
/**
* A 序号
* B 城市
* C 区域
* D 地址
* E 面积
* @return
*/
def run(){
def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
// match(inputfile)
// addPrice(inputfile)
// addArroundPrice(inputfile)
// standard(inputfile)
// addLoudongNum(inputfile)
// addAuthor(inputfile)
// pkPrice(inputfile)
// addPpeValue(inputfile)
// matchMap(inputfile)
// addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
//
// createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
// createPaimai(inputfile,"D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
// createHouse(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
// createDealPrice(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
// createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
}
def addPpeValue(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['F'] && row['R']){
def avg_price = findAvgPrice(row['R'])
if(avg_price){
def ppe15 = Math.abs((row['F'] - avg_price) * 100 / avg_price)
row['Z'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
}
}
});
}
def pkPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
def jsonObject = OpenApiClient.webankEvaluate(dist.city, dist.region, dist.name, row['E'].toString())
if(jsonObject != null && jsonObject.code==200) {
def result = jsonObject.result
if(result){
row['Y'] = result.get("unitPrice")
}
}
}
}
})
}
def match(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
row['Q'] = jsonObject.get("name")
row['R'] = jsonObject.get("id")
}
}
})
}
def matchMap(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['Q'] && row['D']){
def city = row['B']
def address = row['D']
def region = row['C']
println "city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null)
if(jsonObject != null && jsonObject.precise!=-1) {
def propertyType = jsonObject.get("propertyType")
if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){
row['Q'] = jsonObject.get("name")
row['R'] = new ObjectId().toString()
row['W'] = propertyType
def arroundObj = FangjiaClientUtils.arroundPrice(city, address)
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['F'] = priceObj.unitPrice
}
}
}
}
})
}
def addPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['F'] = priceObj.avg_price
}
}
}
})
}
def addArroundPrice(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']) {
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
def priceObj = arroundObj.result
row['F'] = priceObj.unitPrice
row['W'] = "周边均价"
}
}
});
}
def standard(path){
ExcelUtils.multiThreadingModify(path, { row ->
def city = row['B']
def address = row['D']
if(address){
println "[standard] address:"+address
def jsonObject = OpenApiClient.standard2Action(city, address)
if(jsonObject){
println jsonObject
row['G'] = jsonObject.get("province")
row['H'] = jsonObject.get("city")
row['I'] = jsonObject.get("district")
row['J'] = jsonObject.get("township")
row['K'] = jsonObject.get("street")
row['L'] = jsonObject.get("streetNumber")
row['M'] = jsonObject.get("buildingNumber")
row['N'] = jsonObject.get("unitNumber")
row['O'] = jsonObject.get("floor")
row['P'] = jsonObject.get("roomNumber")
}
}
})
}
def addLoudongNum(path){
ExcelUtils.multiThreadingModify(path, { row ->
def commid = row['R']
def houseNum = row['M']
def unitNum = row['N']
println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum
if (commid && houseNum) {
def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum)
if (jsonObject && jsonObject.result) {
if(jsonObject.result.size()>=1){
def obj = jsonObject.result[0]
row['T'] = obj.id
row['S'] = obj.houseNumber
}
}
if(row['P']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
row['V'] = obj.id
row['U'] = obj.roomNum
}
}
}
}
})
}
def createLoudong(path, output){
def rets = new CopyOnWriteArrayList()
def datas = ExcelUtils.read(path, 0, 1)
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
for(def row : datas){
def data = []
data << row['B']
data << row['C']
data << row['D']
data << row['E']
if (row['R'] && row['M']) {
def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N'])
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
def obj = jsonObject.result[0]
data << obj.houseNumber
data << obj.id
data << obj.type// 物业类型
data << ""//主朝向
data << obj.floors//地上层数
data << ""//地下层数
data << obj.buildYear;//竣工日期
data << ""//梯
data << ""//户
data << obj.households//总户数
data << ""//景观
}else{
data << ""
data << ""
data << ""// 物业类型
data << ""//主朝向
data << ""//地上层数
data << ""//地下层数
data << ""//竣工日期
data << ""//梯
data << ""//户
data << ""//总户数
data << ""//景观
}
if(row['P']){
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P'])
if (jsonObject1 && jsonObject1.result) {
if(jsonObject1.result.size()>=1){
def obj = jsonObject1.result[0]
data << obj.roomNum//户号
data << obj.id//户ID
data << obj.direction//主
data << obj.sight//景观
if(obj.roomNum && obj.roomNum.contains("跃")){
data << "跃式"//户型结构
}else{
data << ""//户型结构
}
}
}
}
}
rets << data
}
ExcelUtils.write(output, rets, "底层数据-楼栋-户")
}
def addDetail(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
for(def d : datas){
def commid = d['R']
println "[addDetail] commid:"+commid
if(commid){
def dist = CommonDatautils.findDistrictById(commid)
if(dist){
def row = []
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
if(jsonObject) {
row << d['B']
row << d['C']
row << d['D']
row << jsonObject.get("name")//小区名称
row << commid//小区ID
if(jsonObject.get("alias")){
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
}else{
row << ""
}
row << jsonObject.get("address")//小区地址
row << jsonObject.get("commercialArea")//板块
row << jsonObject.get("township")//街道***********
row << jsonObject.get("avgPrice")//基准价
row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度
row << jsonObject.get("finishBuildTime")//竣工时间
row << jsonObject.get("developer")//开发商名称
row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
row << jsonObject.get("propertyManagement")//物业公司
row << jsonObject.get("managementFee")//物业费
row << jsonObject.get("buildingCount")//住宅幢数
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
}
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(result.keySchool){
row << '是'
}else {
row << ""
}
if(result.school){
row << result.school.join(",").replace('"',"")
}else {
row << ""
}
def bus = []
if(result.bus){
bus.addAll(result.bus)
}
if(result.metro){
bus.addAll(result.metro)
}
if(bus){
row << bus.join(",").replace('"',"")
}else{
row << ""
}
if(result.supermarket){
row << result.supermarket.join(",").replace('"',"")
}else{
row << ""
}
if(result.hospital){
row << result.hospital.join(",").replace('"',"")
}else{
row << ""
}
if(result.bank){
row << result.bank.join(",").replace('"',"")
}else{
row << ""
}
}
rows << row
}
}
}
ExcelUtils.write(output, rows, "底层数据-小区")
}
def createPaimai(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
def keys = []
for(def d : datas){
def commid = d['R']
println "[createPaimai] commid:"+commid
if(commid) {
def dist = CommonDatautils.findDistrictById(commid)
if (dist) {
def key = dist.city+"#"+dist.region+"#"+dist.name
if(keys.contains(key)){
continue
}
keys << key
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100)
if(jsonObject && jsonObject.result){
def results = jsonObject.result
for(def r: results){
def row = []
row << d['B']
row << d['C']
row << d['D']
row << r.matchName
row << r.address
row << r.houseNum
row << r.roomNum
row << r.area
if(r.startDate){
row << r.startDate.substring(0, r.startDate.indexOf("T"))
}else{
row << ""
}
row << r.count
row << r.status
row << r.tradeAvgPrice
row << r.tradePrice
row << r.id
rows << row
}
println dist.name +" "+results.size()
}
}
}
}
ExcelUtils.write(output, rows, "拍卖案例")
}
def createHouse(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid = d['R']
// println "[createHouse] commid:"+commid
if (commid) {
def key = d['Q']
if (keys.contains(key)) {
continue
}
keys << key
def jsonObject = OpenApiClient.houseList(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "挂牌"
row << r.districtName
if(r.estateType){
row << r.estateType
}else{
if(r.purpose){
row << r.purpose
}else{
row << ""
}
}
if(r.room){
def roomStr = r.room+"室"
if(r.hall){
roomStr = roomStr +r.hall+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.area
row << r.fitment
row << r.pDate
row << ""
row << r.avgPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.height){
row << r.height
}else{
row << ""
}
row << r.direction
row << SourceNameUtil.getResourceName(r.root)
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "挂牌案例")
}
def createEvaluation(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid= d['R']
println "[createEvaluation] commid:"+commid
if (commid) {
if (keys.contains(commid)) {
continue
}
keys << commid
def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "评估"
row << r.commName
row << ""
if(r.roomcnt){
def roomStr = r.roomcnt+"室"
if(r.livingroomcnt){
roomStr = roomStr +r.livingroomcnt+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.buildArea
row << r.fitment
row << r.cDate.substring(0, r.cDate.indexOf("T"))
row << ""
row << r.unitPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.floorcnt){
row << r.floorcnt
}else{
row << ""
}
if(r.houseaspect && !r.houseaspect.contains("暂")){
row << r.houseaspect
}else{
row << ""
}
row << "房价网"
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "评估案例")
}
def createDealPrice(path, output){
def datas = ExcelUtils.read(path, 0, 1)
def rows = []
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
def keys = []
for(def d : datas) {
def commid= d['R']
println "[createDealPrice] commid:"+commid
if (commid) {
if (keys.contains(commid)) {
continue
}
keys << commid
def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20)
if(jsonObject && jsonObject.result) {
def results = jsonObject.result
for (def r : results) {
def row = []
row << "成交"
row << r.districtName
if(r.type){
row << r.type
}else{
row << ""
}
if(r.room){
def roomStr = r.room+"室"
if(r.hall){
roomStr = roomStr +r.hall+"厅"
}
row << roomStr
}else{
row << ""
}
row << r.area
row << r.fitment
row << r.tradeDate.substring(0, r.tradeDate.indexOf("T"))
row << ""
row << r.avgPrice
row << r.totalPrice
if(r.floor){
row << r.floor
}else{
row << ""
}
if(r.height){
row << r.height
}else{
row << ""
}
if(r.direction && !r.direction.contains("暂")){
row << r.direction
}else{
row << ""
}
row << r.source
rows << row
}
println d['R'] +" "+results.size()
}
}
}
ExcelUtils.write(output, rows, "成交案例")
}
def addAuthor(def path){
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","穆良书","沈燕芳"]
def personTotal = authors.size()
def avg = Math.ceil(total/personTotal)
def index = 0
def count = 0;
ExcelUtils.modify(path, { row ->
if(row['B']){
if(index < avg){
row['X'] = authors[count]
}
index++
if(index >= avg){
count++
index = 0
}
}
});
}
def findAvgPrice(def commid){
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
def avg_price = null
sqlInstance.eachRow(sql, { r ->
avg_price = r.target_avg_price
});
return avg_price
}
}
@@ -68,6 +68,10 @@ class CommonDatautils {
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -586,8 +590,8 @@ class CommonDatautils {
}
public static def findDistrict(String city, String region, String name, String cat){
public static def findDistrictCount(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
@@ -598,7 +602,24 @@ class CommonDatautils {
q.put("cat", cat)
q.put("status", 0)
def count = getDataStoreConnection().count(q);
return count;
}
public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
q.put("region", region);
}
q.put("name", name);
q.put("cat", cat)
q.put("status", 0)
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def obj : cursor){
@@ -609,6 +630,7 @@ class CommonDatautils {
}
public static def findDistrictList(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
@@ -704,7 +726,7 @@ class CommonDatautils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","ÎâÖз9ÅªÐ¡Çø")
// q.put("name","É­Íû·¼²ÝÂ̰¶")
DBObject keys = new BasicDBObject();
if(fields){
@@ -827,15 +849,24 @@ class CommonDatautils {
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
keys.put("lat2", 1);
keys.put("lng2", 1);
keys.put("name_cat", 1);
keys.put("finish_building_date", 1)
keys.put("mapping_alias", 1)
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
keys.put("sid", 1)
try{
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
return obj;
}
}
}catch(Exception e){
println "error_id:"+id
e.printStackTrace()
}
return null;
}
@@ -1052,9 +1083,16 @@ class CommonDatautils {
public static def queryDistrictById(def id){
def dbo = null
DBObject q = new BasicDBObject();
q.put("_id", new ObjectId(id));
def dbo = getDataStoreConnection().findOne(q);
try{
q.put("_id", new ObjectId(id.trim().toString()));
dbo = getDataStoreConnection().findOne(q);
}catch(Exception e){
println id
}
return dbo;
}
@@ -2384,6 +2422,7 @@ class CommonDatautils {
q.put("block", block);
}
q.put("cat", cat);
q.put("status",0)
def cur = getDataStoreConnection().find(q);
return cur.count();
}
@@ -3040,18 +3079,26 @@ class CommonDatautils {
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate));
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)).limit(1)
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
DBObject dbo;
def prices = []
def index = 0
while(cur.hasNext()) {
dbo = cur.next()
def price = dbo.get("avg_price");
if(price){
break;
def obj = cur.next()
if(index == 0){
dbo = obj
}
prices << obj.get("avg_price");
index++
}
return dbo;
if(prices){
def avg_price = (int)(prices.sum()/prices.size())
dbo.avg_price = avg_price
return dbo;
}
return null;
}
@@ -3166,5 +3213,36 @@ class CommonDatautils {
}
return avg_price;
}
static def findCityAvgRentRadio(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = SQMETRE_RENT_COLL.find(q)
def avg_price = null
def radios = []
for(def d : dbcursor){
if(d.price_to_rent){
radios << d.price_to_rent.toInteger()
}
}
def avg_radios = (int)(radios.sum()/radios.size())
return avg_radios;
}
static def findLoudong(commid){
def q = new BasicDBObject()
q.put("id_in_seaweed", commid)
def count = LOUDONG_LINE_COLL.count(q)
return count;
}
static def findHouseLoudong(commid){
def q = new BasicDBObject()
q.put("district_id", commid)
def count = HOUSE_LOUDONG_COLL.count(q)
return count;
}
}
@@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils;
class DataHandleUtils {
static main(args){
}
static def getStandardRegion(def regionMap, def region){
def newRegion = null;
@@ -94,6 +92,8 @@ class DataHandleUtils {
BigDecimal deSource = new BigDecimal(dSource);
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
}
static def trimChar(String c) {
try {
@@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch {
@Override
public void process(Object args) {
def cities = ["北京", "上海"]
def cities = ["北京", "上海"]//
for (def city : cities) {
def ret = CommonDatautils.getAllDistricts("district", city)
// def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district");
@@ -6,10 +6,16 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.jusfoun.JusfounCommon
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
import java.text.DecimalFormat
import java.util.concurrent.TimeUnit
/**
@@ -19,8 +25,11 @@ import java.util.concurrent.TimeUnit
class DistrictPriceExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def charset = "GBK"
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new DistrictPriceExportBatch(), args)
}
@@ -32,6 +41,9 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createDatas("¸£ÖÝ")
saveToDB(ret)
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){
@@ -76,10 +88,10 @@ class DistrictPriceExportBatch extends BaseBatch{
def createDatas(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", null);
def lastDate = new Date()
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
def dataStr = DateUtils.add(endDateStr, -30)
def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
def lastDate = DateUtils.parseFromTightByPattern(endDateStr, "yyyy-MM-dd")
// List<String> list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202111.csv"))
// def map2 = [:]
// for(def s : list2) {
@@ -88,17 +100,34 @@ class DistrictPriceExportBatch extends BaseBatch{
// map2.put(key, row[5])
// }
def lastMonthPriceMap = findFzDistrictStat("¸£ÖÝ",lastMonthDate)
def result = []
for(def rMap : districts){
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
if(dbo){
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
def curPrice = dbo.get("avg_price")
if(lastMonthPrice && curPrice){
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
if(Math.abs(markup)>3){
def randMarkup = nextDouble(0, 3)
if(markup < 0){
markup = randMarkup*(-1)
}else{
markup = randMarkup
}
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
}
}
def m = []
m << dbo.get("_id").toString()//¾ù¼ÛID
m << rMap.get("_id").toString()//Â¥ÅÌID
m << rMap.city
m << rMap.region
m << rMap.name
m << dbo.get("avg_price")
m << curPrice
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
// if(lastMonthPrice && dbo.get("avg_price")){
// lastMonthPrice = lastMonthPrice.toInteger()
@@ -125,6 +154,19 @@ class DistrictPriceExportBatch extends BaseBatch{
return result
}
def findFzDistrictStat(city,lastMonthDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("month", lastMonthDate)
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
def priceMap = [:]
for(def d : dbcursor){
priceMap.put(d.comm_id, d.avg_price)
}
return priceMap
}
def export(def fileName, def datas){
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
writeCSV(headers, fileName, datas);
@@ -152,5 +194,44 @@ class DistrictPriceExportBatch extends BaseBatch{
file.append(System.getProperty("line.separator"));
}
}
//Ëæ»ú¸¡µãÊý
public static double nextDouble(final double min, final double max) throws Exception {
DecimalFormat df = new DecimalFormat("#.00");
if (max < min) {
throw new Exception("min < max");
}
if (min == max) {
return min;
}
return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
}
def saveToDB(rets){
for(def r : rets){
def q = new BasicDBObject()
q.put("month", curDate.toInteger())
q.put("comm_id", r[1])
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("avg_price", r[5])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def udata = new BasicDBObject()
def o = new BasicDBObject()
o.put("_id", new ObjectId(r[0]))
o.put("comm_id", r[1])
o.put("city", r[2])
o.put("region", r[3])
o.put("name", r[4])
o.put("avg_price", r[5].toInteger())
o.put("month", r[6].toInteger())
FZ_DISTRICT_STAT_COLL.save(o)
}
}
}
}
@@ -8,7 +8,9 @@ import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.rabbitmq.client.impl.AMQImpl
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
/**
* 福州数据结果统计
@@ -17,16 +19,42 @@ class FzDataResultStat {
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
def margeExcel(){
def list2= ExcelUtils.read("D:\\数据\\福州\\价格比对_202204_新2.xlsx", 0 ,1)
def map2 = [:]
for(def s : list2){
def key = s["A"]+"##"+s["B"]+"##"+s["C"]
map2.put(key, s["D"])
}
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\价格比对_202204_新3.xlsx",{row->
if(row["A"]){
def key = row["A"]+"##"+row["B"]+"##"+row["C"]
def price = map2.get(key)
if(price){
row['L'] = price
}
}
})
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
@@ -137,16 +165,13 @@ class FzDataResultStat {
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def price = 0;
def prices = [];
while(cur.hasNext()) {
DBObject dbo = cur.next()
price = dbo.get("avg_price");
if(price){
break;
}
prices << dbo.get("avg_price");
}
return price;
def avg_price = (int)(prices.sum()/prices.size())
return avg_price;
}
@@ -165,4 +190,45 @@ class FzDataResultStat {
weekends = weekends.reverse();
return weekends;
}
def saveCsvData(){
def files = getAllFiles("C:\\Users\\gaojuan\\福州数据(1)")
if(files){
for(def filepath : files){
List<String> list1= FileUtils.readLines(filepath)
println filepath
for(def s : list1){
def row = s.split("@\\|@")
def o = new BasicDBObject()
o.put("_id", new ObjectId(row[0]))
o.put("comm_id", row[1])
o.put("city", row[2])
o.put("region", row[3])
o.put("name", row[4])
o.put("avg_price", row[5].toInteger())
o.put("month", row[6].toInteger())
FZ_DISTRICT_STAT_COLL.save(o)
}
}
}
}
def static getAllFiles(def path) {
File file = new File(path);
File[] tempList = file.listFiles();
def files = [];
for (int i = 0; i < tempList.length; i++) {
if (tempList[i].isFile()) {
files << tempList[i];
}
if (tempList[i].isDirectory()) {
System.out.println("文件夹:" + tempList[i]);
}
}
return files;
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -112,22 +112,28 @@ class TradeExportBatch extends BaseBatch{
// q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate))
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000);
def rets = []
def keys = []
if (cursor != null) {
while (cursor.hasNext()) {
DBObject dbo = cursor.next();
def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name);
if(district){
def ret = [];
ret << dbo._id.toString()
ret << district._id.toString()
ret << dbo.city
ret << dbo.region
ret << dbo.district_name
ret << dbo.avg_price
ret << dbo.area
ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
ret << dbo.total_price
rets << ret;
def tradeDateStr = DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
def key = district._id.toString() +"##" + dbo.avg_price +"##"+dbo.area+"##"+tradeDateStr
if(!keys.contains(key)){
keys << key
def ret = [];
ret << dbo._id.toString()
ret << district._id.toString()
ret << dbo.city
ret << dbo.region
ret << dbo.district_name
ret << dbo.avg_price
ret << dbo.area
ret << tradeDateStr
ret << dbo.total_price
rets << ret;
}
}
}
println "³É½»×ܼƣº"+rets.size()
@@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{
def run(){
def cityMap = CommonDatautils.getAllCityRegions()
cityMap.each{city,regions->
findPriceByCity(city, regions);
findPriceByCity(city, regions);
}
}
@@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{
q.put("region" , r.region)
q.put("month", r.month)
def mom = null
if(lastObj){
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
mom = DataHandleUtils.getDoubleRound(4, mom)
if(lastObj && lastObj.avg_price && r.price){
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
mom = DataHandleUtils.getDoubleRound(4, mom)
}
if(obj){
if(obj.avg_price != r.price){
@@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{
def findEsAvgPrice(city, region){
def detail = [:]
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1])
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0])
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
def must = []
@@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{
if(!priceValue){
priceValue = row['1-metric']
}
if(priceValue){
if(priceValue && priceValue.value){
price = priceValue.value.toInteger()
}
result.price = price;
def date = row.key_as_string.replace("-","").substring(0, 6)
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
if(date!=curMonth){
result.month = date.toInteger()
rets << result
if(date.toInteger()>200001 && result.price){
result.month = date.toInteger()
rets << result
}
}
}
}
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
class OpenApiUtils {
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
address = address.replace("£¨","(").replace("£©",")").replace(" ","")
address = address.replace("£¨","(").replace("£©",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
if(preciseLevelflag){
url = url +"&preciseLevel=strict"
@@ -164,6 +164,7 @@ class OpenApiUtils {
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
return null;
}
address = address.trim().replace(" ","")
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
List<com.alibaba.fastjson.JSONObject> poiList = null;
@@ -48,7 +48,7 @@ class VagueAddressLocationBatch extends BaseBatch {
void process(Object args) {
//"北京","常州",
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
def citys =["±±¾©"] //CommonDatautils.findAllCity();
def citys =["ÖØÇì"] //CommonDatautils.findAllCity();
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
//市谊大厦 华三新村 阳光家园
for (def city : citys) {
@@ -42,17 +42,17 @@ class YouDaDataParser {
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc"
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc"
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2022-02-01"
static def statDate = "2022-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser();//友达二手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -0,0 +1,61 @@
package com.fangjia.script.gj.apiLog
import com.alibaba.excel.util.DateUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.Calendar;
@BatchOwner(Watcher.GaoJuan)
class ApiTimeOutAnalyBatch extends BaseBatch{
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
static main(args) {
runBatch(new ApiTimeOutAnalyBatch(), args);
}
@Override
public void process(Object args) {
def uriCountMap = findTimeOutUri()
sendMessage(uriCountMap)
}
def sendMessage(uriCountMap){
def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:"
def total = message.length()
uriCountMap.each{uri,count->
if(count>=5){
message = message + uri +"\n"
}
}
if(message.length()>total){
println message
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
DingDingMessageUtil.sendTextMessage(access_token, message)
}
}
def findTimeOutUri(){
def q = new BasicDBObject()
q.put("time", new BasicDBObject('$gte', 3000))
def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5)
q.put("c_date", new BasicDBObject('$gte', startTime))
def dbcursor = LOG.find(q)
def uriCountMap = [:]
for(def d : dbcursor){
def count = uriCountMap.get(d.request_uri)
if(!count){
count = 0
}
uriCountMap.put(d.request_uri, count + 1)
}
return uriCountMap
}
}
@@ -0,0 +1,111 @@
package com.fangjia.script.gj.apiLog
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
@BatchOwner(Watcher.GaoJuan)
class ApiUserCountAnalyBatch extends BaseBatch{
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
static main(args) {
runBatch(new ApiUserCountAnalyBatch(), args);
}
@Override
public void process(Object args) {
def rets = findAllUsers()
sendMessage(rets)
}
def sendMessage(def rets){
def message = "【Api接口使用提醒】\n"
def total = message.length()
for(def r : rets){
def type = "正式帐户"
if(r.type!=0){
type = "测试帐户"
}
message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n"
}
if(message.length()>total){
println message
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
DingDingMessageUtil.sendTextMessage(access_token, message)
}
}
def findAllUsers(){
def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'"
def rows = dbFangjia.rows(sql)
def rets = []
for(def row : rows){
def obj = findResources(row.id)
if(obj){
def map = obj.total_count_map
def uris = []
map.each{k,v->
//def visit_count = findVisitCount(row.id, k)
def total_visit_count = findAllVisitCount(row.id, k)
println row.id +" "+k+" "+v+" "+total_visit_count
if(v && total_visit_count){
def precent = v*0.8;
if(total_visit_count>=precent){
uris << k
}
}
}
if(uris){
rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test]
}
}
}
return rets;
}
def findAllVisitCount(userId, uri){
def q = new BasicDBObject()
q.put("user_id", userId.toString())
q.put("uri", uri)
q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date())))
def dbcursor = VISIT_COLL.find(q)
def total = 0
for(def obj : dbcursor){
total = total +obj.visit_count
}
return total
}
def findVisitCount(userId, uri){
def q = new BasicDBObject()
q.put("user_id", userId.toString())
q.put("uri", uri)
q.put("visit_date", DateUtils.truncateDay(new Date()))
def obj = VISIT_COLL.findOne(q)
if(obj){
return obj.visit_count
}
return null
}
def findResources(def userId){
def q = new BasicDBObject()
q.put("user_id", userId)
return AUTHORITY_COLL.findOne(q)
}
}
@@ -18,44 +18,62 @@ class CcbCityPriceStat {
task.run()
}
def run(){
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
def rets = []
def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
def cityRets = []
for(def dist : districts){
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
def matchTotalCount = findMatchTradeCount(city, startDate, endDate)
println city +" "+matchTotalCount
if(matchTotalCount){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29"))
def cityRets = []
for(def dist : districts){
def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01")
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
if(tradeAvgPrice && weekendPrice){
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
cityRets << row
println row.toString()
}
}
print city +" "+ cityRets.size()
rets.addAll(cityRets)
}
print city +" "+ cityRets.size()
rets.addAll(cityRets)
}
println "×ÜÁ¿£º"+rets.size()
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有")
}
def findMatchTradeCount(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("match_id", new BasicDBObject('$exists', true))
def count = DEAL_PRICE_COLL.count(q)
return count
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
// q.put("city", dist.city)
// q.put("region", dist.region)
//
// if(dist.alias){
// if(!dist.alias.contains(dist.name)){
// dist.alias << dist.name
// }
// q.put("district_name", new BasicDBObject('$in', dist.alias))
// }else{
// q.put("district_name", dist.name)
// }
q.put("match_id", dist._id.toString())
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
@@ -199,18 +199,18 @@ class CcbCodeUtils {
static def getBuildStructureCode(def buildStructure){
if(buildStructure){
if(buildStructure.contains("钢结构")){
return "001";
return "3";
}else if(buildStructure.contains("钢、钢筋混凝土结构")){
return "002";
return "1";
}else if(buildStructure.contains("钢筋混凝土结构")){
return "003";
return "1";
}else if(buildStructure.contains("砖木结构")){
return "004";
return "10";
}else if(buildStructure.contains("混合结构")){
return "005";
return "2";
}
}
return "006";
return "11";
}
static def getDistHouseTypeCode(def estateType){
@@ -50,4 +50,32 @@ class FileUtils {
}
out.close();
}
static def createTxtFile(ret, fileName, charset, isAppend){
File file = new File(fileName);
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(row)
sb.append("\n");
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -25,11 +25,11 @@ class DataMatch {
static main(args) {
def task = new DataMatch()
// task.run1()
task.run2()
// task.run2()
// task.run3()
// println task.findAddress("²ıÖݽֵÀÌÄÏã·")
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
}
def run1(){
@@ -0,0 +1,257 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeAppCrawl {
public static final SHORT_DATE = Utils.getLastMonth();
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
public static def SOURCE = "安居客";
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeAppCrawl();
// task.run();
// task.run2()
task.addThirdPrice()
}
def run2(){
def url = "https://m.anjuke.com/sh/community/"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def links = getRegion(url, driver)
for(def link : links){
def doc = getDocument(url, driver)
def str = doc.toString().substring(doc.toString().indexOf("<script> window.__NUXT__"), doc.toString().indexOf("</script>"))
if(str){
str = str.substring(str.indexOf("} ("), str.indexOf("));"))
def arrs = str.split(",")
pritln arrs.size()
}
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def getRegion(url, driver){
//def doc = getDocument(url, driver)
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/fz/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
header.put("sw8-correlation","c291cmNl:YmVpZG91")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
def doc = ABuYunUtil.doGet(url, null, header, "utf-8")
def links = []
if(doc){
def eles = JQueryUtils.select(".column2 a", doc)
for(def e : eles){
def href = e.attr("href")
links << href
}
}
return links;
}
def run(){
//上海:11,北京:14
//"仓山":4071,
// def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
// def regionMap = ["浦东":7,"闵行":11, "松江":15,"宝山":13,"嘉定":14,"徐汇":5,"青浦":19,"静安":2,"普陀":10,"杨浦":9,"奉贤":17,"黄浦":4,"虹口":8,"长宁":6,"金山":18,"崇明":20]
def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678]
def cityMap = ["北京":regionMap]
def cityId = 14
regionMap.each{region, area_id->
for(def i=1;i<40;i++){
def stop = false;
def all_stop = false
def error_count=0
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
//城口
def flagMap = getInfo(stop, i, area_id, cityId, "上海")
stop = flagMap.stop
all_stop = flagMap.all_stop
error_count++
if(error_count>=20){
stop = true
all_stop = true
}
}
if(all_stop){
break;
}
}
}
}
def getInfo(def stop, def i, def area_id, def cityId, def city){
def map = [:]
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
def header = [:]
header.put("accept","application/json, text/plain, */*")
header.put("accept-encoding","gzip, deflate, br")
header.put("accept-language","zh-CN,zh;q=0.9")
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
header.put("referer","https://m.anjuke.com/sh/community/")
header.put("sec-fetch-dest","empty")
header.put("sec-fetch-mode","cors")
header.put("sec-fetch-site","same-origin")
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
header.put("sw8-correlation","c291cmNl:YmVpZG91")
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
def all_stop = false;
def error_count = 0
if (ret) {
JSONObject jsonObj = JSONObject.fromObject(ret);
if(jsonObj && jsonObj.status=="ok" ){
def data = jsonObj.data
if(data && data.total.toInteger()>0){
def communities = data.communities
println "【${i}】 "+communities.size() +" "+url
if(communities.size()>0){
save(city, communities)
Thread.sleep(10000)
stop = true;
}else{
stop = true;
println url +" 返回0"
all_stop = true;
}
}else{
stop = false;
error_count ++
println url +" 无数据"
}
}else{
stop = false;
Thread.sleep(5000)
}
}
map.stop = stop
map.all_stop = all_stop;
return map;
}
def save(city, def ret){
def index = 0
for(def o : ret){
index++
def obj = findIsExist(o.jumpAction)
if(!obj){
def udate = new BasicDBObject()
o.each{k,v->
udate.put(k,v)
}
udate.put("city", city)
udate.put("c_date", new Date())
detail.save(udate)
println index+" [保存]"+o.jumpAction
}else{
println index+" [已存在]"+o.jumpAction
}
}
}
def findIsExist(def url){
def q = new BasicDBObject()
q.put("jumpAction",url)
return detail.findOne(q);
}
def addThirdPrice(){
def q = new BasicDBObject()
def dbcursor = detail.find(q)
def index = 0
for(def d : dbcursor){
index++
def city = d.base.cityName
def region = d.base.areaName
def name = d.base.name
def obj = findByParams(city, region, name);
if(!obj){
def data = new BasicDBObject()
data.put("district_id",d.base.id)
data.put("city", city)
data.put("region", region )
data.put("name", name)
data.put("address",d.base.address)
data.put("avg_price", d.priceInfo.price)
data.put("source", SOURCE)
data.put("s_date", SHORT_DATE)
data.put("c_date", d.c_date)
thirdPartyColl.save(data)
println index +" "+data
}
}
}
def findByParams(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", SOURCE)
q.put("s_date", SHORT_DATE)
return thirdPartyColl.findOne(q);
}
}
@@ -0,0 +1,358 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.WebElement
import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.remote.DesiredCapabilities
import org.openqa.selenium.remote.RemoteWebDriver
class AnjukeDetailCrawl {
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url")
static main(args) {
def task = new AnjukeDetailCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://chongqing.anjuke.com/"
WebDriver driver = new ChromeDriver()
// DesiredCapabilities capabilities = DesiredCapabilities.chrome();
// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL
// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities);
driver.manage().deleteAllCookies()
def regions = getRegions(driver, url)
println "区域链接: "+regions.size()
getAllRegions(driver, regions)
}
def getAllRegions(driver, regions){
for(def url : regions){
def doc = getDocument(url, driver)
def total = getTotal(doc)
if(total) {
if (total > 1250) {
def links = getMoreLinks(url, driver)
println url +" "+links.size()
saveDetailLinks(links)
// parse(links,driver)
} else {
def links = getAllLinks(url, driver)
println url +" "+links.size()
saveDetailLinks(links)
// parse(links,driver)
}
}
}
}
def saveDetailLinks(links){
for(def url : links){
def data = new BasicDBObject()
data.put("city","重庆")
data.put("url", url)
data.put("source", "安居客")
data.put("c_date", new Date())
urlColl.save(data)
}
}
def getMoreLinks(url, driver){
def doc = Jsoup.parse(driver.getPageSource());
def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc)
def links = []
if(pricelinks){
for(def link : pricelinks){
if(link.text()!="不限"){
def urlLink = link.attr("href")
getDocument(urlLink, driver)
// driver.get(urlLink)
links.addAll(getAllLinks(urlLink, driver))
println urlLink +" "+links.size()
}
}
}
return links;
}
def getAllLinks(url, driver){
def stop = false
def urls = []
def doc = Jsoup.parse(driver.getPageSource());
def total = getTotal(doc)
def page = getPage(total, 25)
def pages = []
for(def i=1;i<=page;i++){
pages << url.substring(0, url.size()-1)+"-p"+i+"/"
}
for(def pageUrl : pages){
doc = getDocument(pageUrl, driver)
def details = JQueryUtils.select(".list-cell a.li-row", doc)
for (def d : details) {
def link = d.attr("href")
if(!urls.contains(link)){
urls << d.attr("href")
}
}
}
return urls
}
def getPage(def total, def pageSize){
if(total && total.toInteger()>0){
def page = total.toInteger()/pageSize;
if(page>(int)page){
page = (int)page + 1
return page
}
return (int)page;
}
return null
}
def getTotal(doc){
def totalText = JQueryUtils.selectText(".total-info", doc)
if(totalText){
def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim()
if(total && total.isNumber()){
return total.toInteger()
}
}
return null;
}
def getRegions(driver, url){
// def doc = getDocument(url, driver)
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def els = JQueryUtils.select("a:contains(小区)", doc)
if(els){
def href = els[0].attr("href")
doc = Jsoup.parse(driver.getPageSource());
// driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click()
// driver.findElement(By.linkText("小区")).click()
doc = getDocument(href, driver)
}
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
// if(loginIfm){
// def href = loginIfm[0].attr("src")
// driver.get(href)
// driver.findElementById("pwdTab").click()
// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框
// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框
// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。
// elemUsername.sendKeys("17740818975");
// elemPassword.sendKeys("12qwaszx");
// btn.submit(); // 点击行为;
//
// def regionEles = JQueryUtils.select(".region-parents li a", doc)
// def urls = []
// for(def e : regionEles){
// urls << e.attr("href")
// }
// return urls;
// }
def regionEles = JQueryUtils.select(".region-parents li a", doc)
def urls = []
for(def e : regionEles){
if(e.text()!="不限"){
urls << e.attr("href")
}
}
return urls;
}
def getNontCheckPage(url, driver, doc){
def stop = false;
while(!stop){
if(doc){
}
}
return doc;
}
def parse(def links, driver){
def rets = []
for(def link : links){
def district = parsing(link, driver)
if(district){
// rets << district
// if (rets.size() % 100 == 0) {
saveObj(district)
println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
// rets = []
// }
}
}
if (rets.size()> 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
}
}
def saveData(def results){
def add = 0
for(def ret : results){
saveObj(ret)
}
}
def saveObj(def ret){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = detailColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
detailColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
try{
detailColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
def parsing(url, driver){
def district = [:]
def stop = false
def count = 0
while(!stop){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
def headers = JQueryUtils.select(".crumbs a", doc)
if(headers){
if (headers && headers.size() >= 3) {
district.city = headers[1].text().replace("小区", "")
district.region = headers[2].text().replace("小区", "")
district.name = headers[headers.size() - 1].text().replace("二手房", "")
}
district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc)
def priceStr = JQueryUtils.selectText(".average", doc)
if (priceStr && priceStr.isNumber()) {
district.avg_price = priceStr.toInteger()
}
def infoDiv = JQueryUtils.select(".info-list", doc)
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text())
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
if (houseCountStr) {
houseCountStr = houseCountStr.replace("户", "")
if (houseCountStr) {
district.house_count = houseCountStr.toInteger()
}
}
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text())
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text())
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
if(carSiteStr){
if(carSiteStr.contains("(")){
carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("("))
}
if(carSiteStr && carSiteStr.isNumber()){
district.car_site = carSiteStr
}
}
district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text()
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text())
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
district.url = url
district.c_date = new Date()
district.m_date = new Date()
district.crawl_type = "chrome"
district.root = "anjuke.com"
stop = true;
}else{
Thread.sleep(3000)
count++
}
if(count>10){
stop = true;
}
}
return district
}
String getValue(String s){
if(s && s.contains("暂无")){
return null
}
return s
}
}
@@ -0,0 +1,89 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
class AnjukeDistrictMatch {
static main(args) {
def task = new AnjukeDistrictMatch();
task.run();
// task.run1()
}
def run1(){
def regionPointsMap = findRegionRange("重庆")
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
}
def run(){
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
ExcelUtils.multiThreadingModify(filePath,{row->
if(row['A'] && !row['J']){
def city = row['B']
def region = row['C']
def name = row['D']
def address = row['E']
// def flag = "别名匹配"
// def dist = matchDist(city, null, name)
// if(!row['J'] && name.contains("(")){
// name = name.replace("(${region})","").replace("(","").replace(")","")
// flag = "去括号匹配"
// dist = matchDist(city, region, name)
// }
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
// flag = "映射别名匹配"
// }
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
// if(rets && rets.size()==1){
// dist = rets[0]
// }
// }
//
// if(!dist){
// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district")
// flag = "地址匹配"
// if(dists.size()>1){
// println address +" 地址相同:"+dists.size()
// }else{
// if(dists.size()==1){
// dist = dists[0]
// }
// }
// }
// if(dist){
// row['J'] = flag
// row['K'] = dist._id.toString()
// row['L'] = dist.region
// row['M'] = dist.name
// row['N'] = dist.address
// }
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
if(jsonObject != null && jsonObject.precise!=-1) {
// row['J'] = "匹配接口"
// row['K'] = jsonObject.get("id")
// row['L'] = jsonObject.get("district")
// row['M'] = jsonObject.get("name")
// row['N'] = jsonObject.get("address")
row['M'] = jsonObject.get("name")
}
}
})
}
def matchDist(city, region, name){
def dist = CommonDatautils.findDistrict(city, region, name, "district")
return dist;
}
}
@@ -0,0 +1,189 @@
package com.fangjia.script.gj.crawl
import com.alibaba.fastjson.JSON
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.temp.shenjian.util.MessageUtils
import com.fangjia.script.util.CommonUtils
import org.jsoup.Jsoup
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class AnjukeHouseCrawl {
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
static main(args) {
def task = new AnjukeHouseCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
for(def i=24;i<50;i++){
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
def doc = getDocument(url, driver)
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
println "页数:"+i+" 记录数:"+eles.size()
if(eles){
parser(eles, driver)
}
}
}
def parser(eles, driver){
def index = 0
for(def e : eles){
index ++
def href = e.attr("href")
def doc = getDocument(href, driver)
// parserHtml(doc)
def newUrl = href.substring(0, href.indexOf("?"))
sendMq(newUrl, doc)
println "[ "+index +"] "+ newUrl
}
}
def sendMq(href, doc){
JSON json = new JSONObject()
json.put("project", "sell_anjuke")
json.put("url", href)
JSON result = new JSONObject()
result.put("content", doc.toString())
json.put("result", result)
byte[] packedMsg = MessageUtils.pack(json)
CommonUtils.sendMessage("result2parse", packedMsg)
}
def parserHtml(doc) {
String city = null
def result = [:]
result.siteCode = "安居客"
if (doc) {
result.title = JQueryUtils.selectText([".banner-title h1"], doc)
def heads = JQueryUtils.select(".crumbs a", doc)
if (heads && heads.size() >= 5) {
result.city = heads[0].text().replace("房产网", "")
def orgRegion = heads[2].text().replace("二手房", "")
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
result.block = heads[3].text().replace("二手房", "")
result.district_name = heads[4].text()
} else if (heads && heads.size() >= 3) {
result.city = heads[0].text().replace("房产网", "")
result.district_name = heads[2].text()
def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
if (eles && eles.size() > 0) {
def orgRegion = eles[0].text()
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
}
}
if (!result.district_name) {
println " [未解析出(城市-区域-名称)信息]"
}
println result
def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
if (priceStr) {
if (priceStr.isNumber()) {
result.total_price = priceStr.toDouble() * 10000
}
}
def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
if (roomInfoStr) {
Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
if (room) {
result.room = room
}
if (hall) {
result.hall = hall
}
if (toilet) {
result.toilet = toilet
}
}
def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
if (areaStr) {
result.area = areaStr.toDouble()
}
def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
result.direction = DataCrawlUtils.parseDirection(directionStr)
def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
if (buildDateStr && buildDateStr.contains("/")) {
def arrs = buildDateStr.split("\\/")
if (arrs && arrs.size() == 2) {
result.build_date = arrs[0]
result.estate_type = arrs[1]
}
}
result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
if (result.floor_info && result.floor_info.contains("共")) {
def height = result.floor_info
if (result.floor_info.contains("(")) {
height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
}
height = height.replace("共", "").replace("层", "")
if (height) {
result.height = height.toInteger()
}
}
if (result.area && result.total_price) {
result.avg_price = (result.total_price / result.area).toInteger()
}
def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
}
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(5000)
def els = JQueryUtils.select(".list-main .list-left .list", doc)
if(!els){
els = JQueryUtils.select(".props-main", doc)
}
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -0,0 +1,631 @@
package com.fangjia.script.gj.crawl
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
class AnjukeToDB {
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
static main(args) {
def task = new AnjukeToDB();
task.run();
// task.addNoMatch()
// task.updateEstateType()
// task.gaodeRangeData()
}
def updateEstateType(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1)
for(def row : datas){
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj) {
def type = getEstatType(obj.extend.type)
def commId = row['L']
if(!commId){
commId = row['G']
}
if(type){
if(type=="别墅"){
def udata = new BasicDBObject()
udata.put("estate_type2", type)
udata.put("m_date", new Date())
println commId
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata))
}
}else{
// def udata = new BasicDBObject()
// udata.put("status", 1)
// udata.put("name_cat",11)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata))
}
}
}
}
def gaodeRangeData(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1)
def index = 0
for(def d : datas){
index++
println index
def personName = d['AB']
if(personName){
def commid = d['J']
if(commid){
def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district")
if(obj){
updateDist(d, obj._id.toString())
}
}else{
saveGaode(d)
}
}else{
def commid = d['J']
updateDist(d, commid)
}
}
}
def saveGaode(d){
def oldName = d['E']
def distName = nameFileter(d['C'], oldName)
def udata = new BasicDBObject()
def alias = []
def aliasStr = d['F']
if(aliasStr){
def arrs = aliasStr.split("/")
for(def s : arrs){
alias << s
}
}
if(alias.size()>0){
udata.put("alias", alias)
}
def address = AddressUtil.formatAddress("重庆", d['G'])
if(address){
udata.put("address", address)
}
if(d['N']){
udata.put("virescence_percent", d['N'])
}
if(d['O']){
udata.put("estate_charge", d['O'])
}
if(d['Q']){
udata.put("cubage_percent", d['Q'])
}
if(d['R']){
udata.put("developer", d['R'])
}
if( d['S']){
udata.put("build_area", d['S'])
}
if(d['T']){
def t = d['T'].substring(0, 3)
udata.put("build_date", t)
}
if(d['V'] && d['V'].isNumber()){
udata.put("car_site_count", d['V'].toInteger())
}
if(d['Y']){
def build_type = getBuildType2(d['Y'])
if(build_type){
udata.put("build_type", build_type)
}
}
if(d['Z']){
udata.put("property_life", d['Z'])
}
def lat = null
def lng = null
if(d['H']){
lat = d['H']
lng = d['I']
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
Long dlat = lat.toDouble() * 10000000000L
Long dlng = lng.toDouble() * 10000000000L
udata.put("lat", dlat)
udata.put("lng", dlng)
}
def type = "普通住宅"
if(distName.contains("别墅")) {
type = "别墅"
}else if(distName.contains("商住")){
type = "商住"
}
if(udata){
udata.put("name_cat",0)
udata.put("status",0)
udata.put("cat","district")
udata.put("c_date", new Date())
udata.put("m_date", new Date())
udata.put("city", "重庆")
udata.put("region",d['C'])
udata.put("name", distName)
udata.put("estate_type2",type)
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map",markMap)
DbUtils.seaweed.save(udata);
def stop = false
while (!stop){
if(udata._id.toString()){
addPrice(udata)
stop = true;
}
}
}
}
def updateDist(d, commid){
if(commid){
def dist = CommonDatautils.queryDistrictById(commid)
if(dist){
def oldName = d['E']
def distName = nameFileter(d['C'], oldName)
def udata = new BasicDBObject()
if(!distName.contains("号")){
def alias = dist.alias
if(!alias){
alias = []
}
if(!alias.contains(distName) && dist.name!=distName){
alias << distName
}
def aliasStr = d['F']
if(aliasStr){
def arrs = aliasStr.split("/")
for(def s : arrs){
alias << s
}
}
if(alias.size()>0 && alias!=dist.alias){
udata.put("alias", alias)
}
}
if(dist.address && !dist.address.contains("号")){
def address = AddressUtil.formatAddress("重庆", d['G'])
if(address){
udata.put("address", address)
}
}
if(!dist.virescence_percent && d['N']){
udata.put("virescence_percent", d['N'])
}
if(!dist.estate_charge && d['O']){
udata.put("estate_charge", d['O'])
}
if(!dist.cubage_percent && d['Q']){
udata.put("cubage_percent", d['Q'])
}
if(!dist.developer && d['R']){
udata.put("developer", d['R'])
}
if(!dist.build_area && d['S']){
udata.put("build_area", d['S'])
}
if(!dist.build_date && d['T']){
if(d['T'].size()>=4){
def t = d['T'].substring(0, 4)
udata.put("build_date", t)
}
}
if(!dist.car_site_count && d['V'] && d['V'].isNumber()){
udata.put("car_site_count", d['V'].toInteger())
}
if(!dist.build_type && d['Y']){
def build_type = getBuildType2(d['Y'])
if(build_type){
udata.put("build_type", build_type)
}
}
if(!dist.property_life && d['Z']){
udata.put("property_life", d['Z'])
}
def lat = null
def lng = null
if(d['H']){
lat = d['H']
lng = d['I']
def slat = dist.lat/10000000000.0
def slng = dist.lng/10000000000.0
def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat);
if(distance>1000){
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
Long dlat = lat.toDouble() * 10000000000L
Long dlng = lng.toDouble() * 10000000000L
udata.put("lat", dlat)
udata.put("lng", dlng)
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改] ${dist.name}"+ udata
}
}
}
}
def addNoMatch(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1)
for(def d : datas){
add(d)
}
}
def run(){
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\非人工处理.xlsx",0,1)
for(def data : datas){
// if(data['K']=="镕杰"){
def flag = data['M']
if(flag && flag=="新增"){
// add(data)
}else{
update(data)
}
// }
}
}
def add(def row){
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj){
def type = getEstatType(obj.extend.type)
if(type){
def city = obj.base.cityName
def region = obj.base.areaName
def lat2 = null
def lng2 = null
def lat = null
def lng = null
if(obj.base.lat){
lat2 = obj.base.lat
lng2 = obj.base.lng
JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu")
if(convertObj.getInt("status") == 1) {
String lo = convertObj.get("locations")
lng = Double.parseDouble(lo.split(",")[0])
lat = Double.parseDouble(lo.split(",")[1])
}
}
def distName = nameFileter(obj.base.areaName, obj.base.name)
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
def address = AddressUtil.formatAddress(obj.base.city, obj.base.address)
def dist = CommonDatautils.findDistrict(city, region, distName, "district")
if(!dist){
def data = new BasicDBObject();
data.put("name_cat",0)
data.put("status",0)
data.put("cat","district")
data.put("c_date", new Date())
data.put("m_date", new Date())
data.put("city", city)
data.put("region",region)
data.put("name", distName)
data.put("estate_type2",type)
if(address){
data.put("address", obj.base.address)
}
if(lat){
Long alat = lat*10000000000L
Long alng = lng*10000000000L
data.put("lat", alat)
data.put("lng", alng)
Long alat2 = lat2.toDouble()*10000000000L
Long alng2 = lng2.toDouble()*10000000000L
data.put("lat2",alat2)
data.put("lng2",alng2)
}
data.put("finish_building_date",obj.extend.completionTime)
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){
data.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
}
data.put("build_area", obj.extend.totalArea)
data.put("virescence_percent", obj.extend.landscapingRatio)
data.put("build_type", buildTypeStr)
data.put("business_quarter",obj.base.tradingAreaName)
def markMap = [:]
markMap.put("data_source", "机器入库")
data.put("mark_map",markMap)
DbUtils.seaweed.save(data);
def stop = false
while (!stop){
if(data._id.toString()){
addPrice(data)
stop = true;
}
}
}
}else{
println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type
}
}else{
println anjukeId+" 没找到"
}
}
def update(def row){
def commId = row['L']
if(!commId){
commId = row['G']
}
def dist = CommonDatautils.queryDistrictById(commId)
def anjukeId = row['A']
def obj = findAnjukeDistrictDetail(anjukeId)
if(obj && dist){
def udata = new BasicDBObject()
def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address)
if(address){
if(!dist.address){
if(obj.base.address){
udata.put("address",obj.base.address)
}
}else{
if(dist.address.contains("号")){
if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){
def addr_alias = dist.addr_alias
if(!addr_alias){
addr_alias = []
}
if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){
addr_alias << obj.base.address
udata.put("addr_alias", addr_alias)
}
}
}else{
udata.put("address",obj.base.address)
}
}
}
def distName = nameFileter(obj.base.areaName, obj.base.name)
if(!distName.contains("号")){
def alias = dist.alias
if(!alias){
alias = []
}
if(alias.contains(distName)){
alias << distName
}
if(alias.size()>0 && alias!=dist.alias){
udata.put("alias", alias)
}
}
if(!dist.build_type){
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
if(buildTypeStr){
udata.put("build_type",buildTypeStr)
}
}
if(!dist.household_count){
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){
udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
}
}
if(!dist.finish_building_date){
if(obj.extend.completionTime){
udata.put("finish_building_date",obj.extend.completionTime)
}
}
if(!dist.virescence_percent){
if(obj.extend.landscapingRatio){
udata.put("virescence_percent", obj.extend.landscapingRatio)
}
}
if(!dist.business_quarter){
if(obj.base.tradingAreaName){
udata.put("business_quarter",obj.base.tradingAreaName)
}
}
def lat2 = null
def lng2 = null
if(dist.lat2){
lat2 = dist.lat2/10000000000.0
lng2 = dist.lng2/10000000000.0
}
if(dist.lat && !lat2){
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3)
lng2 = baidu.x
lat2 = baidu.y
Long dLat2 = lat2 * 10000000000L
Long dLng2 = lng2 * 10000000000L
udata.put("lat2", dLat2)
udata.put("lng2", dLng2)
}
if(obj.base.lat) {
def alat2 = obj.base.lat.toDouble()
def alng2 = obj.base.lng.toDouble()
def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2);
if(distance>1000){
println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................"
def lng = null
def lat = null
JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu")
if(convertObj.getInt("status") == 1) {
String lo = convertObj.get("locations")
lng = Double.parseDouble(lo.split(",")[0])
lat = Double.parseDouble(lo.split(",")[1])
}
Long blat2 = alat2 *10000000000L
Long blng2 = alng2 *10000000000L
Long gLat2 = lat *10000000000L
Long gLng2 = lng *10000000000L
udata.put("lat2", blat2)
udata.put("lng2", blng2)
udata.put("lat", gLat2)
udata.put("lng", gLng2)
}
}
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改] ${dist.name}"+ udata
}
}
}
def findAnjukeDistrictDetail(anjukeId){
def q = new BasicDBObject()
try{
q.put("_id",new ObjectId(anjukeId))
return detail.findOne(q)
}catch(Exception e){
print "[查询出错]anjukeId:"+anjukeId
}
return null
}
def nameFileter(def region, def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
name = name.replace("(${region})","").replace("(","").replace(")","")
return name;
}
def getEstatType(type){
if(type){
if(type=="公寓住宅"){
type = "普通住宅"
}else if(type.contains("商住")){
type = "商住"
}else if(type.contains("别墅")){
type = "别墅"
}else{
type = null
}
}
return type
}
def getBuildType(def buildType){
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
def list = []
if(buildType){
def arrs = buildType.split("\\|")
for(def s : arrs){
if(types.contains(s)){
list << s
}
}
if(list){
return list.join(",")
}
}
return null
}
def getBuildType2(def buildType){
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
def list = []
if(buildType){
def arrs = buildType.split(",| ")
for(def s : arrs){
if(types.contains(s)){
list << s
}
}
if(list){
return list.join(",")
}
}
return null
}
def addPrice(dist){
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01")
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25")
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
}
}
}
@@ -0,0 +1,29 @@
package com.fangjia.script.gj.crawl
import org.jsoup.Jsoup
class CommonCrawl {
static def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
}
@@ -0,0 +1,250 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataCrawlUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
str = str.replace("元/m?", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
str = str.replace("m?", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
}
@@ -0,0 +1,347 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 成交数据解析工具类
*/
class DealPriceUtils {
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static {
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
str = "20" + str
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
})
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
})
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
str = str.replaceAll("室|房", '').trim()
return str.toInteger()
})
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
str = str.replace("厅", '').trim()
return str.toInteger()
})
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
str = str.replace("卫", '').trim()
return str.toInteger()
})
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
str = str.replace("万", '').trim()
return (str.toDouble() * 10000).toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
str = str.replace("元/㎡", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m²"), { String str ->
str = str.replace("元/m²", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
str = str.replace("元/平", '').trim()
return str.toDouble().toInteger()
})
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
str = str.replace("元", '').trim()
return str.toDouble().toInteger()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
str = str.replace("平米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
str = str.replace("平方米", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
str = str.replace("平", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
str = str.replace("㎡", '').trim()
return str.toDouble()
})
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m²"), { String str ->
str = str.replace("m²", '').trim()
return str.toDouble()
})
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
str = str.replace("共", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
str = str.replace("总", '').replace('层', '')
return str.toInteger()
})
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n2
}
return null
})
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
def split = str.split("/")
def n1 = split[0].toInteger()
def n2 = split[1].toInteger()
if (n1 <= n2) {
return n1
}
return n1
})
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
str = str.replace("第", "").replaceAll("层|楼", "")
return str.toInteger()
})
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
/**
* 解析成交日期
* @param content
* @return
*/
static Date parseDealDate(String content) {
return parse(content, DEAL_DATE_RESOLVERS)
}
/**
* 解析室数
* @param content
* @return
*/
static Integer parseRoom(String content) {
return parse(content, ROOM_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseHall(String content) {
return parse(content, HALL_RESOLVERS)
}
/**
* 解析厅数
* @param content
* @return
*/
static Integer parseToilet(String content) {
return parse(content, TOILET_RESOLVERS)
}
/**
* 解析总价
* @param content
* @return
*/
static Integer parseTotalPrice(String content) {
return parse(content, TOTAL_PRICE_RESOLVERS)
}
/**
* 解析单价
* @param content
* @return
*/
static Integer parseUnitPrice(String content) {
return parse(content, UNIT_PRICE_RESOLVERS)
}
/**
* 解析面积
* @param content
* @return
*/
static Double parseArea(String content) {
return parse(content, AREA_RESOLVERS)
}
/**
* 解析楼层
* @param content
* @return
*/
static Integer parseFloor(String content) {
return parse(content, FLOOR_RESOLVERS)
}
/**
* 解析楼高
* @param content
* @return
*/
static Integer parseHeight(String content) {
return parse(content, HEIGHT_RESOLVERS)
}
/**
* 格式化小区名
* @param name
* @return
*/
static String formatName(String name) {
name = name.replace("-商住", "").replace("-别墅", "")
//汇福家园平安里(迁)2015-2017
name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "")
//国贸天悦一期2017
name = name.replaceAll("\\d\\d\\d\\d\$", "")
if (name ==~ /.+(\(|()[\d一二三四五六七八九十东南西北|\w]+(期|区)(\)|))$/) {
name = name.replaceAll("\\(|(|\\)|)", '')
} else if (name ==~ /.+(\(|()别墅(\)|))$/) {
name = name.replaceAll("\\(|(|\\)|)", '')
} else if (name ==~ /.+(\(|().+(\)|))$/) {
name = name.replaceAll("(\\(|().+(\\)|))", '')
} else if (name.contains("·") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) {
println("---------------------------->" + name)
name = name.replaceAll("·|-", '')
println(name)
}
if (name ==~ /.{2,}路.{3,}$/ && !(name ==~ /.{2,}路.+号/) && !(name ==~ /.{2,}路.+弄/) && !(name ==~ /.*小区$/)) {
name = name.replaceAll("^.{2,}路", "")
}
return name
}
static Object parse(String content, Map<Pattern, Closure> resolvers) {
if (content != null) {
content = content.replaceAll("\\s", "")
}
for (def resolver : resolvers.entrySet()) {
String match = match(content, resolver.getKey())
if (StringUtils.isNotBlank(match)) {
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
return resolver.getValue()(match)
}
}
return null
}
/**
* 匹配符合条件的字符串
*
* @param str
* @param pattern
* @return
*/
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
static String parsePrice(String priceStr) {
def pattern =Pattern.compile("\\d+(\\.\\d+)?万")
String match = match(priceStr, pattern)
}
static void main(String[] args) {
println parsePrice("顶秀欣园西苑2室1厅1卫492.3万")
}
}
@@ -0,0 +1,167 @@
package com.fangjia.script.gj.crawl
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class ZhugeFangDistCrawl {
public static final SHORT_DATE = Utils.getLastMonth();
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static main(args) {
def task = new ZhugeFangDistCrawl();
task.run();
}
def run(){
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
WebDriver driver = new ChromeDriver()
driver.manage().deleteAllCookies()
getRegions(driver, url)
}
def getRegions(driver, url) {
for(def i=1;i<=11;i++){
def newUrl = url+"page${i}/"
println newUrl
def doc = getDocument(newUrl, driver)
def total = parseList(doc)
// def next = JQueryUtils.select("a:contains(下一页)", doc)
// if(next){
// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
// }
}
}
def parseList(doc){
def city= "福州"
def els = JQueryUtils.select("#listTableBox li", doc)
if (els) {
for(def e : els){
def name = e.select(".house-name").text()
def str = e.select(".house-adr").text()
def address = null
if(str){
if(str.contains("]")){
address = str.substring(str.indexOf("]")+1).trim()
}else{
address = str
}
}
def regionStr = null
if(str && str.contains("]") && str.contains("[")){
regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]"))
}
def region = null
def block = null
if(regionStr){
def arrs = regionStr.split("-")
if(arrs && arrs.size()>0){
if(arrs.size()==2){
block = arrs[1].trim()
}
if(arrs.size()>=1){
region = arrs[0].trim()
}
}
}
def links = e.select(".house-name a")
def href = null
if(links){
href = links[0].attr("href")
}
def priceStr = e.select(".price-number").text()
def price = null
if(priceStr && !priceStr.contains("暂无")){
if(priceStr.contains("元")){
priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim()
}
if(priceStr && priceStr.isNumber()){
price = priceStr.toInteger()
}
}
def obj = findByParams(city, region, name)
if(obj){
if(price){
DBObject udata2 = new BasicDBObject()
udata2.put("avg_price", price)
udata2.put("m_date", new Date())
println "[upeate] ${region} "+name
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
}
}else{
def udata = new BasicDBObject()
udata.put("city", city)
udata.put("region", region)
udata.put("block", block)
udata.put("url", href)
udata.put("address", address)
udata.put("name", name)
udata.put("avg_price", price)
udata.put("s_date", SHORT_DATE)
udata.put("c_date", new Date())
udata.put("m_date", new Date())
udata.put("source", "诸葛找房")
thirdPartyColl.save(udata)
println "[save] ${region} "+name+" "+priceStr
}
}
}
return els.size()
}
def findByParams(city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("source", "诸葛找房")
q.put("s_date", SHORT_DATE)
return thirdPartyColl.findOne(q);
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
Thread.sleep(10000)
def els = JQueryUtils.select("#listTableBox li", doc)
if(els && els.size()>0){
stop = true;
}else{
error++
driver.manage().deleteAllCookies()
println "[${error}] error: "+url
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=20){
stop = true;
}
}
return doc
}
}
@@ -55,7 +55,7 @@ class AddDistrictFieldValue {
// }
// })
// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
def citys = ["重庆"]//CcbCommonUtils.citys;
//第三方均价数据匹配
// for(def city : citys){
// def rets = findAllDataByCity(city)
@@ -63,11 +63,11 @@ class AddDistrictFieldValue {
// }
//第三方均价数据填补
// for(def city : citys){
//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
// def rMap = getMatchDistrict(city)
// createResultValue(dists, rMap)
// }
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
def rMap = getMatchDistrict(city)
createResultValue(dists, rMap)
}
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
@@ -103,52 +103,52 @@ class AddDistrictFieldValue {
// }
def citys = ["上海"]
def updateIndex = 0
def rets = []
rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
def total = dists.size()
for(def dist : dists){
if(dist.household_count){
def ret = []
ret << dist._id.toString()
ret << dist.city
ret << dist.region
ret << dist.name
ret << dist.household_count
ret << dist.loudong_count
def details = findDistrictDetail(dist)
if(details){
for(def d : details){
def region = RegionData.formatRegion(city, d.region)
if(d.region == region){
if(dist.household_count && dist.household_count==d.house_count){
ret << d.loudong_count
ret << "链家"
// updateIndex++
// def udata = new BasicDBObject()
// udata.put("household_count", d.house_count.toInteger())
// udata.put("m_date", new Date())
// def q = new BasicDBObject()
// q.put("_id", dist._id)
// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}else{
ret << ""
ret << ""
}
rets << ret
}
}
ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
}
// def citys = ["上海"]
// def updateIndex = 0
// def rets = []
// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
// def total = dists.size()
//
// for(def dist : dists){
// if(dist.household_count){
// def ret = []
// ret << dist._id.toString()
// ret << dist.city
// ret << dist.region
// ret << dist.name
// ret << dist.household_count
// ret << dist.loudong_count
// def details = findDistrictDetail(dist)
// if(details){
// for(def d : details){
// def region = RegionData.formatRegion(city, d.region)
// if(d.region == region){
// if(dist.household_count && dist.household_count==d.house_count){
// ret << d.loudong_count
// ret << "链家"
//// updateIndex++
//// def udata = new BasicDBObject()
//// udata.put("household_count", d.house_count.toInteger())
//// udata.put("m_date", new Date())
//// def q = new BasicDBObject()
//// q.put("_id", dist._id)
//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
// }
// }
// }
// }else{
// ret << ""
// ret << ""
// }
//
// rets << ret
// }
// }
// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
// }
}
@@ -303,6 +303,14 @@ class AddDistrictFieldValue {
def createResultValue(dists, rMap){
def updateIndex = 0
def total = dists.size()
def addressIndex = 0
def allAreaIndex = 0
def buildAreaIndex = 0
def virescence_percent_index = 0
def cubage_percent_index = 0
def finish_building_date_index = 0
def household_count_index = 0
def car_site_count_index = 0
for(def d : dists){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
@@ -311,12 +319,14 @@ class AddDistrictFieldValue {
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
addressIndex++
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
allAreaIndex++
udata.put("all_area", lastValue)
}
}
@@ -324,6 +334,7 @@ class AddDistrictFieldValue {
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
buildAreaIndex++
udata.put("build_area", lastValue)
}
}
@@ -331,6 +342,7 @@ class AddDistrictFieldValue {
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
virescence_percent_index ++
udata.put("virescence_percent", lastValue)
}
}
@@ -338,6 +350,7 @@ class AddDistrictFieldValue {
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
cubage_percent_index ++
udata.put("cubage_percent", lastValue)
}
}
@@ -345,6 +358,7 @@ class AddDistrictFieldValue {
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
finish_building_date_index ++
udata.put("finish_building_date", lastValue)
}
}
@@ -352,6 +366,7 @@ class AddDistrictFieldValue {
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
household_count_index ++
udata.put("household_count", lastValue)
}
}
@@ -359,6 +374,7 @@ class AddDistrictFieldValue {
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
car_site_count_index ++
udata.put("car_site_count", lastValue)
}
}
@@ -373,6 +389,15 @@ class AddDistrictFieldValue {
}
}
}
println "addressIndex "+ addressIndex
println "allAreaIndex "+ allAreaIndex
println "buildAreaIndex "+ buildAreaIndex
println "virescence_percent_index "+ virescence_percent_index
println "cubage_percent_index "+ cubage_percent_index
println "finish_building_date_index "+ finish_building_date_index
println "household_count_index "+ household_count_index
println "car_site_count_index "+ car_site_count_index
}
def getValuesByField(sourceMap, field, estateType){
@@ -38,8 +38,8 @@ class FangjiaClientUtils {
}
public static def sendDistrictPriceCalculate(def city, def region, def name, def count){
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"&region="+region+"&retainHistory=true"
def content = null;
try{
content = new HttpRequest().newGet(url, null, "utf-8")
@@ -83,6 +83,23 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100"
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static Map<String, Object> JsonToMap(JSONObject json) {
Map<String, Object> columnValMap = new HashMap<String, Object>();
@@ -172,7 +172,8 @@ class NameProcess {
if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){
return null
}
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){
//|| name.endsWith("组")
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){
return null
}
if(name.contains("、")||name.contains("|")||name.contains(".")){
@@ -58,6 +58,8 @@ class DistrictIndexBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM")
static main(args) {
runBatch(new DistrictIndexBatch(), args);
}
@@ -69,9 +71,10 @@ class DistrictIndexBatch extends BaseBatch{
def run(){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def citys = CommonDatautils.findAllCity();
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
//def citys = CommonDatautils.findAllCity();
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
def code = cityCodeDatas.get(city)
def city_avg_total_price = null
if(city!="ÉϺ£"){
@@ -131,7 +134,7 @@ class DistrictIndexBatch extends BaseBatch{
map.total_price = total_price
//map.comp_value = potentiality
//map.avg_total_price = total_price_index
map.month = "2022-03"
map.month = month
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
@@ -229,7 +232,7 @@ class DistrictIndexBatch extends BaseBatch{
map.comp_value = potentiality
map.avg_total_price = total_price_index
map.month = "2022-03"
map.month = month
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
map.finish_date = year
@@ -240,6 +243,15 @@ class DistrictIndexBatch extends BaseBatch{
if(rMap.sell_avg_price && sqmetre_rent){
map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger()
}
if(d.household_count){
map.household_count = d.household_count
}
if(d.lat){
def lMap = [:]
lMap.lon = d.lng/10000000000.0
lMap.lat = d.lat/10000000000.0
map.location = lMap
}
def newUrl = targetUrl +"_doc/"+map.id
httpSaveData(newUrl, map)
@@ -560,7 +572,7 @@ class DistrictIndexBatch extends BaseBatch{
return datas
}
def sortByDistance(d, list){
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
@@ -0,0 +1,123 @@
package com.fangjia.script.gj.location
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
class DistrictMapMatch {
static main(args) {
def task = new DistrictMapMatch()
task.mapMatch()
}
def mapMatch(){
def index = 0
ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d->
if(!d['H']){
index++
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// if (!map || !map.lat) {
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
// }
if(map){
d['F'] = map.id
if(map.adname){
d['G'] = map.adname
}else if(map.region){
d['G'] = map.region
}
d['H'] = map.name
d['I'] = map.address
if(map.type){
d['J'] = map.type
}else if(map.estateType){
d['J'] = map.estateType
}
d['K'] = map.lng
d['L'] = map.lat
d['M'] = map.score
d['N'] = map.source
}
}
},0)
}
def findStandardObject2(def city, def region, def name, def address, def rets) {
if (rets) {
def bestObj = [:]
if (rets.size() >= 1) {
for (def mapObj : rets) {
def score = 0
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
score = 0.2
}
if(mapObj.name && name){
def newName = mapObj.name;
def names = []
if(mapObj.name && mapObj.name.indexOf("(")>-1){
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
names << name1
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
names << name2
}else{
names << newName
}
def maxNameScore = 0
for(def n : names){
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
if(score1 && score1>0){
def nameScore = (score1 * 4)/10
if(nameScore > maxNameScore){
maxNameScore = nameScore
}
}
}
if(maxNameScore){
try{
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
}catch(Exception e){
maxNameScore = 0
}
}
score = maxNameScore + score;
}
if(mapObj.address && address){
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
if(addressScore && addressScore>0){
try{
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
}catch(Exception e){
addressScore = 0
}
score = addressScore*4/10 + score
}
}
if(!bestObj){
mapObj.score = score
bestObj = mapObj
}else{
if(score > bestObj.score){
mapObj.score = score
bestObj = mapObj
}
}
}
}
return bestObj
}
return null
}
}
@@ -0,0 +1,402 @@
package com.fangjia.script.gj.meituan
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.gj.paimai.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.bson.types.ObjectId
class MatchTest {
static main(args) {
def task = new MatchTest()
// task.run()
// task.addNum()
// task.dongguang()
// task.meituanDel()
task.addLevel()
}
def addLevel() {
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
ExcelUtils.multiThreadingModify(path, { row ->
def name = row['D']
def address = row['E']
def city = RegionData.formatCity(row['A'])
if(name && !row['O']){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
// if(rMap.level){
// row['Q'] = rMap.level
// }
}
}
}
})
}
def meituanDel(){
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
def index = 0
ExcelUtils.modify(path, { row ->
def flag = row['O']
if(flag){
def city = row['F']
def region = row['G']
def name = nameFileter(row['H'])
def obj = null
if(city == "东莞"){
obj = CommonDatautils.findDistrict(city, null, name, "district")
}else{
obj = CommonDatautils.findDistrict(city, region, name, "district")
}
if(obj){
index++
println index +" "+ city +' '+ region+" "+name
def udata = new BasicDBObject()
udata.put("name_cat", 11)
udata.put("status", 1)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", new ObjectId(obj._id.toString()))
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
},0);
}
def dongguang(){
def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx
ExcelUtils.modify(path, { row ->
def region = null
if(row['B'] && row['B']=="NULL" ){
region = null
}
def name = nameFileter(row['H'])
def address = row['E']
if (name && name.contains("湖畔新家")) {
def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district")
if(obj){
def json = GaoDeApiUtils.geo(row['F'], address)
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
def twinshop = getStreetInfo(m.location)
if(twinshop){
def newRegion = RegionData.formatRegion("东莞",twinshop)
if(newRegion){
def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district")
if(dbObj){
println "delete"
DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id))
}else{
println "update"
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
}
}
}
// println address +" "+twinshop
}
}
}
}
},0)
}
def getStreetInfo(def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
}
}
}
return township;
}
def addNum(){
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx
def index = 0
ExcelUtils.modify(path, { row ->
index++
row['A'] = index
},0)
}
def run() {
def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]//
def weekend = DateUtils.getFirstDayOfWeek(new Date());
def cityLevelMap = [:]
def radioMap = [:]
for(def city : citys){
// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
// radioMap.put(city, avg_radio)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
def datas = addPriceData(dists, priceMap);
datas = calcuteLevel(datas);
cityLevelMap.put(city, ["price":priceMap, "level":datas])
println city +" 楼盘评级同步完成"
}
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
ExcelUtils.multiThreadingModify(path, { row ->
def region = null
if(row['C'] && row['C']=="NULL" ){
region = null
}
def name = row['D']
def address = row['F']
def city = RegionData.formatCity(row['B'])
if(name && !row['N']){
if(city){
def map = cityLevelMap.get(city)
def rMap = null
if(map){
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
}
if(rMap){
if(rMap.rank){
row['Q'] = rMap.rank
}
if(rMap.level){
row['P'] = rMap.level
}
}
}
}
if (name && row['N']) {
//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name)
// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null)
// if (jsonObject != null && jsonObject.precise != -1) {
// row['F'] = jsonObject.get("city")
// row['G'] = jsonObject.get("district")
// row['H'] = jsonObject.get("name")
// row['I'] = jsonObject.get("propertyType")
// row['J'] = jsonObject.get("map")
// row['K'] = jsonObject.get("lng")
// row['L'] = jsonObject.get("lat")
// }
if(row['G'] && row['H'] && row['I']){
// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H'])
// if(priceObj){
// row['M'] = priceObj.avg_price
// }
// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H'])
// if(sqmetreRentPrice){
// row['N'] = sqmetreRentPrice
// }
if(row['N'] && !row['O']){
def avg_rent_radio = radioMap.get(city)
row['O'] = (int)(row['N']/avg_rent_radio)
}
}
}
})
}
def getArroundDistrictPriceByAddress(def city, def address, def levels){
def json = GaoDeApiUtils.geo(city, address)
def location = null
def detail = new BasicDBObject()
def level = null
def rank = null
if(json && json.count>=1){
def m = json.geocodes[0]
if(m){
location = m.location
level = m.level
}
}
if(location){
def arrs = location.split(",")
detail.lng = arrs[0].toString().toDouble()*10000000000L
detail.lat = arrs[1].toString().toDouble()*10000000000L
}
if(detail.lng){
detail.city = city
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
if(list && list.size()>10){
list = sortByDistance(detail, list);
list = list[0..9]
}
def arroundAvgPrice = getArroundAvgPrice(list)
if(arroundAvgPrice){
def mod = Integer.MAX_VALUE;
for(def d : levels){
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
if(newMod<mod){
mod = newMod
rank = d.level
}
}
}
}
def m = [:]
m.level = level
m.rank = rank
return m
}
def sortByDistance(d, list){
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
}
//排序
Collections.sort(list, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return list;
}
def addPriceData(dists, priceMap){
def newLists = []
for(def d : dists){
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
if(price){
d.avg_price = price;
newLists << d;
}
}
if(newLists){
newLists.sort {a,b->
def v1 = a.avg_price
def v2 = b.avg_price
return v1.compareTo(v2)
}
}
return newLists;
}
def calcuteLevel(def datas){
def total = datas.size()
def v = (int)(total/10)
def start = 0;
def end = 0;
def map = [:]
for(int i=1;i<=10;i++){
end = end + v;
if(i == 10){
map.put(i, [start, total])
}else{
map.put(i, [start, end])
}
start = end;
}
for(def d=0;d<datas.size();d++){
def dist = datas[d]
for (def o in map) {
def ranges = o.value
def m = ranges[0]
def n = ranges[1]
if(d==0){
if(d<=n){
dist.level = 10 - o.key + 1
break
}
}else{
if(d>m && d<=n){
dist.level = 10 - o.key + 1
break
}
}
}
}
return datas;
}
def getArroundAvgPrice(def list){
def prices = []
for(def l : list){
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
if(priceObj){
prices << priceObj.avg_price
}
}
if(prices){
def avg_price = (prices.sum()/prices.size()).toInteger()
return avg_price;
}
return null
}
def nameFileter(def name){
name = name.replace("·","").replace(".","").replace("|","")
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
return name;
}
}
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
static main(args) {
def task = new BeiKeCrawlByChrome();
// task.run()
task.run()
// task.run1()
task.addDealRegion();
}
@@ -45,9 +45,21 @@ public class BeiKeCrawlByChrome {
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def cityMap = getAllCityUrls(driver)
def citys = getAllCityUrls(driver)
citys = citys.reverse()
def start = 0
// for(def i=0;i<citys.size();i++){
// def m = citys[i]
// println i+" "+ m.city
// if(m.city=="天津"){
// start = i
// break;
// }
//
// }
// citys = citys[(start+1)..(citys.size()-1)]
getLogin(driver)
crawlData(driver, cityMap)
crawlData(driver, citys)
}
@@ -85,13 +97,13 @@ public class BeiKeCrawlByChrome {
}
}
def crawlData(driver, cityMap){
def crawlData(driver, citys){
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
cityMap.each{city, cityUrl->
for(def m : citys){
//def detailUrls = getPageUrls(driver, cityUrl)
def detailUrls = getPageDeail(driver, city, cityUrl)
println cityUrl +" 详情链接:"+detailUrls.size()
def detailUrls = getPageDeail(driver, m.city, m.url)
println m.city +" 详情链接:"+detailUrls.size()
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
}
}
@@ -110,7 +122,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -141,7 +153,7 @@ public class BeiKeCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -150,6 +162,10 @@ public class BeiKeCrawlByChrome {
break;
}
}
if(page>=100){
stop = true;
break;
}
page++;
}else{
stop = true;
@@ -177,12 +193,12 @@ public class BeiKeCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
}
}
def areaStr = arrs[2].replace("平米", "")
if (areaStr) {
if (areaStr && areaStr.isNumber()) {
detail.area = areaStr.toDouble()
}
@@ -199,7 +215,8 @@ public class BeiKeCrawlByChrome {
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
}
detail.source = "贝壳找房"
def dbo = findTrade(detail)
def detailUrl = e.select(".title a").attr("href")
def dbo = findTrade(detailUrl)
if (!dbo) {
def houseInfo = e.select(".houseInfo").text()
def houseInfos = houseInfo.split("\\|")
@@ -212,7 +229,7 @@ public class BeiKeCrawlByChrome {
detail.height = floorStr.toInteger()
}
}
detail.url = e.select(".title a").attr("href")
detail.url = detailUrl
detail.create_date = new Date()
detail.m_date = new Date()
println detail
@@ -297,8 +314,8 @@ public class BeiKeCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger()
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -320,13 +337,14 @@ public class BeiKeCrawlByChrome {
}
def findTrade(def detail){
def findTrade(def detailUrl){
def q = new BasicDBObject()
q.put("city", detail.city)
q.put("region", detail.region)
q.put("district_name", detail.district_name)
q.put("area", detail.area)
q.put("total_price", detail.total_price)
// q.put("region", detail.region)
// q.put("district_name", detail.district_name)
// q.put("area", detail.area)
// q.put("total_price", detail.total_price)
q.put("url", detailUrl)
q.put("trade_date", detail.trade_date)
q.put("source", detail.source)
def dbo = DEAL_PRICE_COLL.findOne(q)
@@ -337,17 +355,21 @@ public class BeiKeCrawlByChrome {
driver.get("https://www.ke.com/city/")
Thread.sleep(8000)
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
def list = []
if(cityEles){
for(def e : cityEles){
def href = e.attr("href")
if(!e.text().contains("南京")){
cityMap.put(e.text(), "https:"+href)
}
// if(e.text().contains("福州")){
def cityMap = [:]
cityMap.city = e.text()
cityMap.url = "https:"+href
list << cityMap
//cityMap.put(e.text(), "https:"+href)
// }
}
}
return cityMap
return list
}
def getLogin(driver){
@@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver
import org.openqa.selenium.support.ui.ExpectedConditions
import org.openqa.selenium.support.ui.WebDriverWait
import javax.swing.JApplet
import java.text.DateFormat
import java.util.function.Consumer
@@ -41,45 +42,138 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
System.setProperty("http.nonProxyHosts", "localhost");
def doc = getDocument();
def detailUrls = getDetailLinks(doc);
def rets = batchParse(detailUrls);
saveToDB(rets);
WebDriver driver = new ChromeDriver()
def proviceMap = getDocument(driver);
def citys = getCity(driver,proviceMap)
def detailUrls = getDetailLinks(driver, citys);
def getDetailLinks(doc){
def urls = []
def getDetailLinks(driver, cityList){
def zxcitys = ["天津市","北京市","上海市","重庆市"]
for(def cityMap : cityList){
def citys = cityMap.citys
citys.each{city,id->
def urls = []
if(!zxcitys.contains(cityMap.provice)){
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
getDocument(url, driver)
}else{
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
getDocument(url, driver)
}
def doc = Jsoup.parse(driver.getPageSource());
def count = JQueryUtils.selectText(".count", doc)
def totalPage = getTotalPage(count, 12)
if(totalPage){
for(def i=1;i<=totalPage;i++){
doc = Jsoup.parse(driver.getPageSource());
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
if(eles){
for(def el :eles){
def href = el.attr("href")
def map = [:]
map.province = cityMap.provice
map.city = city
map.url = "https://paimai.caa123.org.cn"+href
urls << map
}
}
try{
driver.findElement(By.className("next")).click()
}catch(Exception e){
Thread.sleep(8000)
driver.findElement(By.className("next")).click()
}
}
def rets = batchParse(urls, driver);
saveToDB(rets);
println city +" "+urls.size()
}
}
println cityMap.provice +" 完成"
}
//return urls;
}
def getTotalPage(def total, def pageSize){
if(total){
def mode = total.toInteger()/pageSize
if(mode > mode.toInteger()){
mode = mode.toInteger() +1
}
return mode;
}
return 0;
}
def getCity(driver, proviceMap){
def rows = []
def zxcitys = ["天津市","北京市","上海市","重庆市"]
proviceMap.each {k,v->
if(!zxcitys.contains(k)){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
def doc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc)
def cityMap = [:]
for(def c : cityEles){
def cityid = c.attr("data-cityid")
def city = c.text()
if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
cityMap.put(city, cityid)
}
}
def map = [:]
map.provice = k
map.provice_id = v
map.citys = cityMap
rows << map
}else{
def map = [:]
map.provice = k
map.provice_id = v
def cityMap = [:]
cityMap.put(k, v)
map.citys = cityMap
rows << map
}
}
return rows
}
def getDocument(driver){
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport=");
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
def doc = Jsoup.parse(driver.getPageSource());
def provinceMap = [:]
if(doc){
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
if(eles){
for(def el :eles){
def href = el.attr("href")
urls << "https://paimai.caa123.org.cn"+href
def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc)
for(def ele : proviceEles){
def provice = ele.text()
def value = ele.attr("data-proviceid")
if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){
provinceMap.put(provice, value)
}
}
}
return urls;
return provinceMap;
}
def getDocument(){
WebDriver driver = new ChromeDriver()
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
WebDriverWait wait = new WebDriverWait(driver,20)
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
def batchParse(def urls, driver){
def doc = Jsoup.parse(driver.getPageSource());
return doc;
}
def batchParse(def urls){
WebDriver driver = new ChromeDriver()
def ret = []
for(def url :urls){
driver.get(url);
def doc = Jsoup.parse(driver.getPageSource());
def res = parse(url, doc);
for(def map :urls){
def doc = getDocument(map.url, driver)
def res = parse(map.province, map.city, map.url, doc);
if(res){
ret << res
}
@@ -87,111 +181,167 @@ def batchParse(def urls){
return ret;
}
def parse(url,doc){
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def parse(province, city, url,doc){
if(doc){
def result = [:]
// def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
// result.address = address
if(result.address){
def rMap = OpenApiUtils.matchDistrict(result.address)
if(rMap) {
result.matchCity = rMap.matchCity
result.matchRegion = rMap.matchRegion
result.matchName= rMap.matchName
result.matchAddress = rMap.matchAddress
result.province = province
result.city = city
def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
result.address = address
if(result.address && !result.address.contains("测试")) {
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
if(finishDateStr) {
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss");
}
result.status = JQueryUtils.selectText([".bd_state"], doc)
result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc)
result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc)
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",","")
result.tradePrice = curPriceStr.toInteger()
}
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
if (finishDateStr) {
try{
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
}catch(Exception e){
}
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
if(evaluationPriceStr) {
evaluationPriceStr = evaluationPriceStr.replace(",", "")
if(evaluationPriceStr.isInteger()) {
result.evaluationPrice = evaluationPriceStr.toInteger()
}
}
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
if(depositPriceStr) {
depositPriceStr = depositPriceStr.replace(",", "")
if(depositPriceStr.isInteger()) {
result.depositPrice = depositPriceStr.toInteger()
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
if (curPriceStr) {
curPriceStr = curPriceStr.replace(",", "")
result.tradePrice = curPriceStr.toDouble().toInteger()
}
}
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
if(addPriceStr) {
addPriceStr = addPriceStr.replace(",", "")
if(addPriceStr.isInteger()) {
result.addPrice = addPriceStr.toInteger()
}
}
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
if(startPriceStr) {
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}
}
result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc)
result.retain = JQueryUtils.selectText(["#retain"], doc)
def imgs =JQueryUtils.select(".sf-pic-slide img", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("src")
if(href.contains("http")) {
imgLinks << href
}else {
imgLinks << url.substring(0, url.indexOf(":")+1)+href
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
if (evaluationPriceStr) {
evaluationPriceStr = evaluationPriceStr.replace(",", "")
if (evaluationPriceStr.isNumber()) {
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
}
}
result.imgUrls = imgLinks
}
def desc = JQueryUtils.selectText(["#J_desc"], doc)
if(!desc){
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
}
if(desc) {
if(desc.contains("别墅")) {
result.address = desc.substring(0, desc.indexOf("别墅")+2)
}else if(desc.contains("号楼")) {
result.address = desc.substring(0, desc.indexOf("号楼")+2)
}else if(desc.contains("房屋")) {
result.address = desc.substring(0, desc.indexOf("房屋")+2)
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
if (depositPriceStr) {
depositPriceStr = depositPriceStr.replace(",", "")
if (depositPriceStr.isNumber()) {
result.depositPrice = depositPriceStr.toDouble().toInteger()
}
}
}
def descr = JQueryUtils.selectText(["body"], doc)
if(descr) {
def area = getAreaNumber(descr, 5000);
if(area){
result.area = area.toDouble()
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
if (addPriceStr) {
addPriceStr = addPriceStr.replace(",", "")
if (addPriceStr.isNumber()) {
result.addPrice = addPriceStr.toDouble().toInteger()
}
}
}
if(result.area && result.tradePrice){
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
}
result.content = DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
result.url = url
return result;
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
if (startPriceStr) {
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
if (startPriceStr.isNumber()) {
result.startPrice = startPriceStr.toDouble().toInteger()
}
}
result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc)
result.retain = JQueryUtils.selectText(["#retain"], doc)
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc)
if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
}
try{
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim())
}catch(Exception e){
}
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
if (imgs) {
def imgLinks = []
for (def img : imgs) {
def href = img.attr("src")
if (href.contains("http")) {
imgLinks << href
} else {
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
}
}
result.imgUrls = imgLinks
}
def desc = JQueryUtils.selectText(["#J_desc"], doc)
if (!desc) {
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
}
if (desc) {
if (desc.contains("别墅")) {
result.address = desc.substring(0, desc.indexOf("别墅") + 2)
} else if (desc.contains("号楼")) {
result.address = desc.substring(0, desc.indexOf("号楼") + 2)
} else if (desc.contains("房屋")) {
result.address = desc.substring(0, desc.indexOf("房屋") + 2)
}
}
def descr = JQueryUtils.selectText(["body"], doc)
if (descr) {
def area = getAreaNumber(descr, 5000);
if (area) {
result.area = area.toDouble()
}
}
if (result.area && result.tradePrice) {
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
if(result.startDate){
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
}
result.content = DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
result.url = url
return result;
}
}
return null
}
@@ -207,11 +357,13 @@ def saveToDB(rets){
ret.each{k,v->
obj.put(k,v)
}
obj.source= "中拍网"
if(dbo){
obj.remove("url")
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
obj.c_date = new Date()
obj.m_date = new Date()
paimaiResultColl.save(obj)
}
}
@@ -229,16 +381,25 @@ def getAreaNumber(input, maxArea) {
def area = null;
for (def val : list) {
if(val.toString().contains(" ")){
val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size())
}
if(val.contains("室")){
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
}
def v = val.replace("ƽ·½Ã×", "").replace("©O", "").replace("ƽ", "").replace("·½Ã×", "").replace(",", "").replace("m2", "");
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
if(v && v.toString().isNumber()){
if (maxArea) {
if (v.toDouble() < maxArea) {
area = v;
break;
}
} else {
area = v
break;
}
} else {
area = v
break;
}
}
if (area) {
@@ -0,0 +1,311 @@
package com.fangjia.script.gj.paimai
import org.apache.commons.codec.binary.Base64
import org.apache.commons.lang.StringUtils
import sun.font.Font2D
import sun.font.Font2DHandle
import sun.font.TrueTypeFont
import sun.font.TrueTypeGlyphMapper
import java.awt.Font
import java.awt.font.FontRenderContext
import java.awt.font.GlyphVector
import java.awt.geom.AffineTransform
import java.lang.reflect.Field
import java.util.regex.Matcher
import java.util.regex.Pattern
class DataUtil {
static def checkMinDate(curDate, checkDate){
if(curDate){
def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
if(curDate.getTime()<minDate.getTime()){
return null
}
}
return curDate
}
static def checkMaxDate(curDate, checkDate){
if(curDate){
def maxDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
if(curDate.getTime()>maxDate.getTime()){
return null
}
}
return curDate
}
static def getAreaNumber(def input, def maxArea) {
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)')
Matcher matcher = pattern.matcher(input)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() > 0) {
def area = null
for (def val : list) {
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "")
if (maxArea) {
if (v.isDouble()) {
if (v.toDouble() < maxArea) {
area = v
break
}
}
} else {
area = v
break
}
}
if (area) {
return area
}
}
return null
}
static def getPaiMaiType(def s) {
if (s) {
if (s.toString().contains("结束")) {
return "已完成"
} else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) {
return "进行中"
} else if (s.toString().contains("即将开始")) {
return "即将开始"
} else if (s.toString().contains("流拍")) {
return "流拍"
} else if (s.toString().contains("撤回")) {
return "撤回"
} else if (s.toString().contains("暂缓")) {
return "暂缓"
} else if (s.toString().contains("中止")) {
return "中止"
} else if (s.toString().contains("失败")) {
return "失败"
}
}
return s
}
static def getDateRange(def s) {
s = s.replace("起", "")
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
def str = matcher.group()
list.add(str)
// if(str) {
// def indx1 = str.indexOf("年")
// def indx2 = str.indexOf("月")
// def month = str.substring(indx1+1, indx2);
// if(month.toInteger()<12) {
// list.add(str);
// }
// }
}
if (list.size() >= 1) {
String s1 = list.get(0)
s1 = s1.replace("法院将于", "").replace(" ", "")
def array = s1.split("至")
return array
}
return null
}
static def getStartPrice(def s) {
Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() >= 1) {
String s1 = list.get(0)
if (s1.contains("万元")) {
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","")
return (int) (s1.toDouble() * 10000)
} else {
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","")
if (s1.isInteger()) {
return s1.toInteger()
} else {
return (int) (s1.toDouble())
}
}
}
return null
}
static def getEstateType(def s) {
Pattern pattern = Pattern.compile('房屋类型:.*;')
Matcher matcher = pattern.matcher(s)
List<String> list = new ArrayList()
while (matcher.find()) {
list.add(matcher.group())
}
if (list.size() >= 1) {
String s1 = list.get(0)
if (s1) {
s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";"))
return s1.replace(")", "").replace("】", "")
}
}
return null
}
static def findEstateType(def s) {
if (s) {
def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]
for (def type : villTypes) {
if (s.contains(type)) {
return "别墅"
}
}
def districtTypes = ["住宅", "公寓"]
for (def type : districtTypes) {
if (s.contains(type)) {
return "普通住宅"
}
}
}
return null
}
static def filterChar(def detailMap) {
def newDetailMap = [:]
detailMap.each { def k, v ->
if (k.toString().contains(".")) {
newDetailMap.put(k.toString().replace(".", "\\u002e"), v)
} else {
newDetailMap.put(k, v)
}
}
return newDetailMap
}
/**
* 匹配符合条件的字符串
*
* @param str
* @param pattern
* @return
*/
static String match(String str, Pattern pattern) {
if (str == null) {
return null
}
Matcher matcher = pattern.matcher(str)
if (matcher.find()) {
return matcher.group()
}
return null
}
/**
* font-secret字符串专用解密工具
*
* @param key 密匙
* @param encodeString 加密后的字符串
* @return 解密后的字符串
*/
static String decodeString(String key, String encodeString) {
if (!key) {
return ""
}
try {
//base64解码,初始化字体
Base64 base64 = new Base64()
byte[] ss = base64.decodeBase64(key)
InputStream inputStream = new ByteArrayInputStream(ss)
Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream)
FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false)
GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "")
//获取font中字形的映射关系,字段为private,使用反射
Class<?> clazz = Font.class
Field[] fs = clazz.getDeclaredFields()
Font2DHandle font2DHandle = null
for (int i = 0; i < fs.length; i++) {
fs[i].setAccessible(true)// 将目标属性设置为可以访问
if (fs[i].getName().equals("font2DHandle")) {
font2DHandle = (Font2DHandle) fs[i].get(dynamicFont)
}
}
//得到映射关系
Font2D font2D = font2DHandle.font2D
TrueTypeFont trueTypeFont = (TrueTypeFont) font2D
TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper()
//开始解密,encodeString为加密后的字符串
StringBuffer buffer = new StringBuffer()
char[] chars = encodeString.toCharArray()
for (int i = 0; i < chars.length; i++) {
buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1)
}
return buffer.toString()
} catch (Exception e) {
e.printStackTrace()
}
return ""
}
static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"]
static String urlToSiteName(String url) {
if (StringUtils.isEmpty(url)) {
return null
}
for (String suffix : SITE_SUFFIXS) {
if (url.indexOf(suffix) > -1) {
String s = StringUtils.substringBefore(url, suffix)
if (s.indexOf(".") > -1) {
s = StringUtils.substringAfterLast(s, ".")
if (StringUtils.isNotBlank(s)) {
return s + suffix
}
} else if (s.indexOf("://") > -1) {
s = StringUtils.substringAfterLast(s, "://")
if (StringUtils.isNotBlank(s)) {
return s + suffix
}
} else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
return s + suffix
}
return null
}
}
return null
}
static double getRound(int scale, double dSource) {
BigDecimal deSource = new BigDecimal(dSource)
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue()
}
static def filterEmpty(def s){
if(s && s.contains("暂无")){
return null
}
return s
}
}
@@ -0,0 +1,348 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import javafx.scene.input.DataFormat
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import org.openqa.selenium.By
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
class IcbcCrawl {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new IcbcCrawl();
task.run();
}
def run(){
def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this"
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
driver.get(url)
driver.findElementById("category_SM310000000000000006").click()
Thread.sleep(3000)
def urls = getAllDetailLinks(driver)
println "共:"+urls.size()
detailCrawling(urls, driver)
}
def getAllDetailLinks(driver){
def stop = false
def urls = []
while (!stop){
def doc = Jsoup.parse(driver.getPageSource());
def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc)
if(eles){
for(def e : eles){
def href = e.select(".li_2_name a").attr("href")
urls << "https://gf.trade.icbc.com.cn"+href
}
def nextPage = JQueryUtils.select("a:contains(下一页)", doc)
if(nextPage){
driver.findElement(By.linkText("下一页")).click();
}else{
stop = true
}
}else{
stop = true
}
// if(urls.size()>=10){
// stop = true
// }
}
return urls;
}
def detailCrawling(def urls, driver){
def rets = []
for(def url : urls){
driver.get(url)
Thread.sleep(5000)
def doc = Jsoup.parse(driver.getPageSource());
def result = parser(doc, url)
if (result) {
rets << result
println rets.size()
}
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
rets = []
}
}
if(rets.size()>0){
saveData(rets)
print "完"
}
}
def parser(doc, url){
def result = [:]
if(doc){
def title = JQueryUtils.selectText("#prodnameb", doc)
def id = url.substring(url.indexOf("="), url.length())
if(title){
def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc)
if(cityStr && cityStr.contains("...")){
def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc)
if(cityEles){
cityStr = cityEles[0].attr("title")
}
}
if(cityStr && cityStr.contains("省")){
result.province = cityStr.substring(0, cityStr.indexOf("省"))
result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length())
}else{
if(cityStr.contains("自治区")){
result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3)
result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length())
}else{
result.city = cityStr.substring(0, cityStr.indexOf("市"))
}
}
if(!result.city){
return null
}
result.title = title
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","")
def t1 = JQueryUtils.selectText("#pricep", doc)
if(t1 && t1.contains("原因")){
result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length())
}
result.type = JQueryUtils.selectText("dl:contains(类  型:) dd", doc)
def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc)
if(enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def lookCount = JQueryUtils.selectText(["#visitorCount"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
}
def callCount = JQueryUtils.selectText(["#remindPPNum"], doc)
if(callCount) {
result.callCount = callCount.toInteger()
}
def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim()
if(curPriceStr.isInteger()) {
result.curPrice = curPriceStr.toInteger()
}else {
result.curPrice = (int)(curPriceStr.toDouble())
}
}
def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc)
if(startPriceEles) {
def startPriceStr = startPriceEles[0].attr("title")
if(startPriceStr){
startPriceStr = startPriceStr.replace(",", "").replace("¥","")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}else {
result.startPrice = (int)(startPriceStr.toDouble())
}
}
}
def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
if(addPriceEles){
def addPriceStr = addPriceEles[0].attr("title")
if(addPriceStr){
addPriceStr = addPriceStr.replace(",", "").replace("¥","")
if(addPriceStr) {
result.addPrice = addPriceStr.toDouble().toInteger()
}
}
}
def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc)
if(evaluationPriceEles){
def evaluationPriceStr = evaluationPriceEles[0].attr("title")
if(evaluationPriceStr){
evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","")
if(evaluationPriceStr.isInteger()) {
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
}else {
result.evaluationPrice = (int)(evaluationPriceStr.toDouble())
}
}
}
def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
if(depositPriceEles){
def v = depositPriceEles[0].attr("title")
if(v){
v = v.replace(",", "").replace("¥","")
if(v.isInteger()) {
result.depositPrice = v.toDouble().toInteger()
}else {
result.depositPrice = (int)(v.toDouble())
}
}
}
result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc)
result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc)
result.court = JQueryUtils.selectText("#goToCourt", doc)
def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc)
if(mobileEles){
result.mobile = mobileEles[0].attr("title")
}
def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc)
if(s1) {
def start = s1.indexOf("法院将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
}
def dateRangs = PaimaiDataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
}
}
def trs = JQueryUtils.select([".rygtable table tr"], doc)
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
if(d_tds[2].text() && d_tds[2].text()!="/"){
detailMap.put(d_tds[1].text(), d_tds[2].text())
}
}else {
if(d_tds[1].text() && d_tds[1].text()!="/") {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}
}
}
if(detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if(!result.address) {
result.address = detailMap.get("拍品名称")
if(!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs =JQueryUtils.select("#scroll li img", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("src")
if(href.contains("icbc.com")) {
imgLinks << "https:"+href
}
}
result.imgUrls = imgLinks
}
def detailStr = JQueryUtils.selectText(".detail_list", doc)
if(detailStr){
def area = DataUtil.getAreaNumber(detailStr, 5000)
if(area) {
result.area = area.toDouble()
if(result.tradePrice) {
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
}
}
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = detailStr.toString()
result.url = url
result.mDate = new Date()
result.source = "融e够"
return result;
}
}
return null
}
def getCallCount(id){
def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id
def s = ABuYunUtil.doPost(url, null, null, "utf-8", true)
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
println jsonObject
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
}
@@ -0,0 +1,540 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.openqa.selenium.WebDriver
import org.openqa.selenium.chrome.ChromeDriver
import javax.sound.midi.MidiSystem
import java.util.regex.Pattern
class JdPaimaiCrawl {
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
static main(args) {
def task = new JdPaimaiCrawl();
task.run();
}
def run(){
def existsIds = []//findAllIds()
println "已存在:"+existsIds.size()
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
System.setProperty("http.nonProxyHosts", "localhost");
WebDriver driver = new ChromeDriver()
def links = getList(driver)
def ids = getDetailLinks(links, driver)
println "共:"+ids.size()
parserList( existsIds,ids, driver)
}
def getList(driver){
def list = []
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268"
// for(def i=1;i<=32;i++){
// if(i==8){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
// }else if(i==5){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
// }else if(i==17){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
// }else if(i==19){
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
// }else{
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
// }
// }
def links = []
for(def url : list){
driver.get(url)
def doc = Jsoup.parse(driver.getPageSource());
//def document = ABuYunUtil.doPost(url, null, null, "utf-8", true)
if(doc){
def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")"))
JSONObject obj = JSONObject.fromObject(json);
if(obj && obj.total){
def total = obj.total
def page = total/40
def p = page.toInteger()
if(page > p){
p = p +1
}
for(def i=1;i<=p;i++){
links << url+"&page="+i
}
// if(links){
// break;
// }
}
}
}
return links;
}
def getDetailLinks(links, driver){
def rets = []
def citys = ["北京","上海","天津","重庆"]
for(def link : links){
driver.get(link)
def doc = Jsoup.parse(driver.getPageSource());
if(doc) {
def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")"))
JSONObject obj = JSONObject.fromObject(json);
if (obj && obj.ls) {
for(def o : obj.ls){
def map = [:]
if(citys.contains(o.province)){
map.province = o.province
map.city = o.province
map.region = o.city
}else{
map.province = o.province
map.city = o.city
}
map.id = o.id.toString()
rets << map
}
}
}
}
return rets
}
def parserList(existsIds,list, driver){
def rets = []
for(def map : list) {
if(!existsIds.contains(map.id)){
def url = "https://paimai.jd.com/" + map.id
def doc = getDocument(url, driver)
if (doc) {
def result = parsering(doc, url, map.province, map.city, map.region)
if (result) {
rets << result
}
if (rets.size() % 100 == 0) {
def add= saveData(rets)
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
rets = []
}
}
}
}
if(rets.size()>0){
saveData(rets)
print "完"
}
}
def saveData(def results){
def add = 0
for(def ret : results){
def q = new BasicDBObject()
q.put("url", ret.url)
def dbo = paimaiResultColl.findOne(q);
DBObject obj = new BasicDBObject();
ret.each{k,v->
obj.put(k,v)
}
if(dbo){
obj.m_date = new Date()
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
}else{
add++
obj.c_date = new Date()
obj.m_date = new Date()
try{
paimaiResultColl.save(obj)
}catch(Exception e){
println obj.url
}
}
}
return add
}
def parsering(doc, def url, def province, def city, def region){
def result = [:]
if(doc) {
result.proince = province
result.city = city
if(region){
result.region = region
}
def title = JQueryUtils.selectText([".pm-name"], doc)
result.title = title
result.address = JQueryUtils.selectText([".pm-location em"], doc)
if(result.address) {
//调用获取城市
// if(!city) {
// city = RegionData.formatCity(cityMap.name)
// }
if(city) {
def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true)
if (rMap) {
result.matchCity = rMap.city
result.matchRegion = rMap.region
result.matchName = rMap.name
result.matchAddress = rMap.address
result.houseNum = rMap.house_num
result.roomNum = rMap.room_num
}
}
}
result.status = JQueryUtils.selectText([".pm-summary .state"], doc)
def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc)
if(enrollCount) {
result.enrollCount = enrollCount.toInteger()
}
def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc)
if(lookCount) {
result.lookCount = lookCount.toInteger()
}
def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc)
if(callCount) {
result.callCount = callCount.toInteger()
}
def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc)
if(curPriceStr) {
curPriceStr = curPriceStr.replace(",", "")
if(curPriceStr.isInteger()) {
result.curPrice = curPriceStr.toInteger()
}else {
result.curPrice = (int)(curPriceStr.toDouble())
}
}
def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc)
if(startPriceStr) {
startPriceStr = startPriceStr.replace(",", "")
if(startPriceStr.isInteger()) {
result.startPrice = startPriceStr.toInteger()
}else {
result.startPrice = (int)(startPriceStr.toDouble())
}
}
def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc)
if(tradePriceStr) {
tradePriceStr = tradePriceStr.replace(",", "")
if(tradePriceStr.isInteger()) {
result.tradePrice = tradePriceStr.toInteger()
}else {
result.tradePrice = (int)(tradePriceStr.toDouble())
}
}
def finishDateStr = JQueryUtils.selectText([".endtime"], doc)
if(finishDateStr && finishDateStr.contains("结束时间")) {
finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" "))
result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss")
}
def lis = JQueryUtils.select(".description ul li", doc)
if(lis && lis.size()) {
for(def li : lis) {
def str = li.select("span").text()
if(str.contains("起拍")) {
def v= li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
if(v.isInteger()) {
result.startPrice = v.toDouble().toInteger()
}else {
result.startPrice = (int)(v.toDouble())
}
}
}else if(str.contains("评估价")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v.isInteger()) {
result.evaluationPrice = v.toDouble().toInteger()
}else {
result.evaluationPrice = (int)(v.toDouble())
}
}else if(str.contains("加价幅度")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
result.addPrice = v.toDouble().toInteger()
}
}else if(str.contains("保证金")) {
def v = li.select("em").text().replace("¥", "").replace(",", "")
if(v) {
if(v.isInteger()) {
result.depositPrice = v.toDouble().toInteger()
}else {
result.depositPrice = (int)(v.toDouble())
}
}
}else if(str.contains("延时周期")) {
result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "")
}else if(str.contains("优先竞买权人")) {
result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "")
}
}
}
def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc)
if(trs) {
def detailMap = [:]
for(def tr : trs) {
def d_tds = tr.select("td")
if(d_tds && d_tds.size()>=2) {
if(d_tds.size()==3) {
detailMap.put(d_tds[1].text(), d_tds[2].text())
}else {
detailMap.put(d_tds[0].text(), d_tds[1].text())
}
}else {
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
}
}
}
if(detailMap) {
result.detailMap = DataUtil.filterChar(detailMap)
}
if(!result.address) {
result.address = detailMap.get("拍品名称")
if(!result.address) {
result.address = detailMap.get("标的名称")
}
}
}
def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc)
if(imgs) {
def imgLinks = []
for(def img : imgs) {
def href = img.attr("href")
if(href.contains("http")) {
imgLinks << href
}else {
imgLinks << url.substring(0, url.indexOf(":")+1)+href
}
}
result.imgUrls = imgLinks
}
}
//解析面积
def article = JQueryUtils.selectText([".floor-content"], doc)
if(article) {
def area = DataUtil.getAreaNumber(article, 5000)
if(!area) {
article = JQueryUtils.selectText(["body"], doc)
if(article) {
area = DataUtil.getAreaNumber(article, 5000)
}
}
if(area) {
result.area = area.toDouble()
if(result.tradePrice) {
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
}
}
}
def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc)
if(s1) {
def start = s1.indexOf("将于")
def end = s1.indexOf("止")
if(start>-1 && end>-1 && end>start){
s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止"))
}
def dateRangs = DataUtil.getDateRange(s1)
if(dateRangs && dateRangs.size()==2) {
for(def j=0;j<dateRangs.size();j++) {
def date = dateRangs[j]
def pattern = null
if(date.endsWith("日时")||date.endsWith("日")) {
pattern = "yyyy年MM月dd日"
}else {
def ind = date.indexOf("日")
if(date.indexOf("时")==-1 && date.toString().length()>ind+1) {
date = date +"时"
}
pattern = "yyyy年MM月dd日HH时"
}
if(j==0) {
try{
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
}catch(Exception e){
}
}else {
if(!result.finishDate) {
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
}
}
}
}
if(!result.startPrice) {
result.startPrice = DataUtil.getStartPrice(s1)
}
def estateType = DataUtil.findEstateType(s1)
if(estateType) {
result.matchEstateType = estateType
}
}
if(!result.startDate){
def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","")
result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日")
}
if(result.finishDate){
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
}
if(result.startDate){
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
}
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
result.url = url
result.city = city
result.mDate = new Date()
result.source = "京东"
return result;
}
def getDocument(url, driver){
def stop = false
def error = 0
def doc = null
while(!stop){
try{
driver.get(url)
doc = Jsoup.parse(driver.getPageSource());
if(doc){
stop = true;
}else{
error++
Thread.sleep(5000)
}
}catch(Exception e){
error++
}
if(error>=10){
stop = true;
}
}
return doc
}
def findAllIds(){
def q = new BasicDBObject()
q.put("url", Pattern.compile(".*jd.com.*"))
def dbcursor = paimaiResultColl.find(q)
def ids = []
for(def d : dbcursor){
def url = d.url
if(url){
def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length())
ids << id
}
}
return ids;
}
}
@@ -37,7 +37,27 @@ public class LianjiaCrawlByChrome {
task.run()
// task.run1()
task.addDealRegion();
// task.standardRegion()
task.standardRegion()
// task.updadteDealPriceField()
}
def updadteDealPriceField(){
def citys = CommonDatautils.findAllCity()
def index = 0
for(def city : citys){
index ++
def q = new BasicDBObject()
q.put("city", city)
q.put("hall", "0")
def count = DEAL_PRICE_COLL.count(q)
println "[${index}] "+city +" "+count
if(count && count>0){
def udata = new BasicDBObject()
udata.put("hall", 0)
DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
println "[ÐÞ¸Ä]"+city +" "+ count
}
}
}
def run1(){
@@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
}else{
@@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome {
def durl = e.select(".title a").attr("href")
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
parse(city, e)
@@ -227,6 +247,10 @@ public class LianjiaCrawlByChrome {
break;
}
}
if(page>=100){
stop = true;
break;
}
page++;
}else{
stop = true;
@@ -254,7 +278,7 @@ public class LianjiaCrawlByChrome {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
}
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
}
}
@@ -374,8 +398,8 @@ public class LianjiaCrawlByChrome {
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
if (roomStr) {
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ"))
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")).toInteger()
}
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
@@ -416,12 +440,14 @@ public class LianjiaCrawlByChrome {
def cityDoc = Jsoup.parse(driver.getPageSource());
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
def cityMap = [:]
def citys= ["¸£ÖÝ"]
if(cityEles){
for(def e : cityEles){
for(def e : cityEles) {
def href = e.attr("href")
// if(!href.contains("fang.lianjia") && e.text().contains("¸£ÖÝ")){
if (!href.contains("fang.lianjia")){
cityMap.put(e.text(), href)
}
}
}
return cityMap
}

Some files were not shown because too many files have changed in this diff Show More