Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -0,0 +1,202 @@
|
||||
package com.fangjia.core.util;
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class SeaweedUtils {
|
||||
public static void main(String[] args){
|
||||
System.out.println(formatVirescencePercent("1"));
|
||||
}
|
||||
/**
|
||||
* 标准化输出绿化率
|
||||
*/
|
||||
public static String formatVirescencePercent(String virescence_percent) {
|
||||
// 绿化率
|
||||
if (StringUtils.isNotBlank(virescence_percent) && checkValue(virescence_percent)!=null) {
|
||||
double val = Double.valueOf(virescence_percent);
|
||||
if (val < 1) {
|
||||
return String.valueOf(val * 100);
|
||||
}
|
||||
if (val == 1) {
|
||||
return String.valueOf(val * 10);
|
||||
}
|
||||
if (val >= 70) {
|
||||
return "65";
|
||||
}
|
||||
return String.valueOf(val);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标准化容积率输出
|
||||
*/
|
||||
public static String formatCubagePercent(String cubage_percent) {
|
||||
if (StringUtils.isNotBlank(cubage_percent) && checkValue(cubage_percent)!=null) {
|
||||
double val = Double.valueOf(cubage_percent);
|
||||
if (val >= 7 && val < 8) {
|
||||
return "6.2";
|
||||
}
|
||||
if (val >= 8 && val < 9) {
|
||||
return "6.3";
|
||||
}
|
||||
if (val >= 9 && val < 10) {
|
||||
return "6.5";
|
||||
}
|
||||
if (val >= 10) {
|
||||
return "6.7";
|
||||
}
|
||||
return String.valueOf(val);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标准化车位输出
|
||||
*/
|
||||
public static String formatCarSiteCount(String car_site_count) {
|
||||
if (StringUtils.isNotBlank(car_site_count) && checkValue(car_site_count)!=null) {
|
||||
long val = Long.valueOf(car_site_count);
|
||||
if (val > 2500 || val < 100) {
|
||||
return "";
|
||||
}
|
||||
return String.valueOf(val);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标准化户数输出
|
||||
*/
|
||||
public static String formatHouseholdCount(String household_count) {
|
||||
if (StringUtils.isNotBlank(household_count) && checkValue(household_count)!=null ) {
|
||||
if (Integer.valueOf(household_count) < 50) {
|
||||
return null; // 户数
|
||||
}
|
||||
return household_count;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 标准化总占地面积输出
|
||||
*/
|
||||
public static String formatAllArea(String all_area) {
|
||||
if (StringUtils.isNotBlank(all_area)) {
|
||||
Double formattedVal = 0.0;
|
||||
if (all_area.toString().indexOf("公顷") != -1) {
|
||||
String val = all_area.toString();
|
||||
formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000);
|
||||
} else if(checkValue(all_area)!=null) {
|
||||
formattedVal = Double.valueOf(all_area);
|
||||
}
|
||||
if (formattedVal > 100000 || formattedVal == 0.0) {
|
||||
return null;
|
||||
}
|
||||
return String.valueOf(formattedVal);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 提取浮点数与整数
|
||||
*/
|
||||
static String getNumber(String input) {
|
||||
Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static Float checkValue(String value){
|
||||
if(StringUtils.isNotBlank(value)){
|
||||
value = getNumber(value);
|
||||
if(StringUtils.isNotBlank(value)){
|
||||
value = subZeroAndDot(value);
|
||||
if(isNumber(value)){
|
||||
Float v = Float.parseFloat(value);
|
||||
if(v>0){
|
||||
return v;
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
}else if(isFloat(value)){
|
||||
Float v = Float.parseFloat(value);
|
||||
if(v>0){
|
||||
return v;
|
||||
}else{
|
||||
return null;
|
||||
}
|
||||
} else{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static boolean isNumber(String str){
|
||||
try {
|
||||
Integer.parseInt(str);
|
||||
} catch (NumberFormatException e) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static boolean isFloat(String str){
|
||||
try {
|
||||
Float.parseFloat(str);
|
||||
} catch (NumberFormatException e) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* 正则表达式去掉多余的.与0
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static String subZeroAndDot(String v){
|
||||
|
||||
if(StringUtils.isNotBlank(v)){
|
||||
if(v.indexOf(".")!=-1){
|
||||
Double myNum2 = Double.valueOf(v);
|
||||
BigDecimal b = new java.math.BigDecimal(myNum2);
|
||||
Double myNum3 = b.setScale(2, java.math.BigDecimal.ROUND_HALF_UP).doubleValue();
|
||||
v = myNum3.toString();
|
||||
}
|
||||
|
||||
if(v.indexOf(".")>-1){
|
||||
while(true)
|
||||
{
|
||||
if(v.endsWith("0")){
|
||||
v = v.substring(0,v.length()-1);
|
||||
}else{
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(v.endsWith(".")){
|
||||
v=v.substring(0,v.length()-1);
|
||||
}
|
||||
}
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
package com.fangjia.dal.mongo.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.model.BsEvaluation;
|
||||
|
||||
public interface BsEvaluationDao extends MongoDao<BsEvaluation> {
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
package com.fangjia.dal.mongo.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.mongo.model.BsEvaluation;
|
||||
|
||||
public class BsEvaluationDaoImpl extends MongoDaoImpl<BsEvaluation> implements
|
||||
BsEvaluationDao {
|
||||
}
|
||||
@@ -99,6 +99,9 @@ public class PaiMai implements MongoEntity {
|
||||
private Integer marketAvgPrice;
|
||||
/**物业类型*/
|
||||
private String matchEstateType;
|
||||
private String houseNum;
|
||||
private String unitNum;
|
||||
private String roomNum;
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@@ -488,4 +491,28 @@ public class PaiMai implements MongoEntity {
|
||||
public void setMatchEstateType(String matchEstateType) {
|
||||
this.matchEstateType = matchEstateType;
|
||||
}
|
||||
|
||||
public String getHouseNum() {
|
||||
return houseNum;
|
||||
}
|
||||
|
||||
public void setHouseNum(String houseNum) {
|
||||
this.houseNum = houseNum;
|
||||
}
|
||||
|
||||
public String getUnitNum() {
|
||||
return unitNum;
|
||||
}
|
||||
|
||||
public void setUnitNum(String unitNum) {
|
||||
this.unitNum = unitNum;
|
||||
}
|
||||
|
||||
public String getRoomNum() {
|
||||
return roomNum;
|
||||
}
|
||||
|
||||
public void setRoomNum(String roomNum) {
|
||||
this.roomNum = roomNum;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
package com.fangjia.dal.mongo.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.*;
|
||||
import java.util.Date;
|
||||
|
||||
@Entity
|
||||
@Table(name = "business.evaluation")
|
||||
public class BsEvaluation implements MongoEntity {
|
||||
private String id;
|
||||
private String fullAddress;
|
||||
private String commid;
|
||||
private String cityName;
|
||||
private String distName;
|
||||
private String commName;
|
||||
private String address;
|
||||
private Double buildArea;
|
||||
private Integer floor;
|
||||
private Integer floorcnt;
|
||||
private Integer roomcnt;
|
||||
private Integer livingroomcnt;
|
||||
private String houseaspect;//ÄÏ
|
||||
private String fitment;
|
||||
private Date cDate;
|
||||
private Integer unitPrice;
|
||||
private Integer totalPrice;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.AUTO)
|
||||
@Column(name = "_id")
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "full_address")
|
||||
public String getFullAddress() {
|
||||
return fullAddress;
|
||||
}
|
||||
|
||||
public void setFullAddress(String fullAddress) {
|
||||
this.fullAddress = fullAddress;
|
||||
}
|
||||
|
||||
@Column(name = "commid")
|
||||
public String getCommid() {
|
||||
return commid;
|
||||
}
|
||||
|
||||
public void setCommid(String commid) {
|
||||
this.commid = commid;
|
||||
}
|
||||
|
||||
@Column(name = "cityname")
|
||||
public String getCityName() {
|
||||
return cityName;
|
||||
}
|
||||
|
||||
public void setCityName(String cityName) {
|
||||
this.cityName = cityName;
|
||||
}
|
||||
|
||||
@Column(name = "distname")
|
||||
public String getDistName() {
|
||||
return distName;
|
||||
}
|
||||
|
||||
public void setDistName(String distName) {
|
||||
this.distName = distName;
|
||||
}
|
||||
|
||||
@Column(name = "commname")
|
||||
public String getCommName() {
|
||||
return commName;
|
||||
}
|
||||
|
||||
public void setCommName(String commName) {
|
||||
this.commName = commName;
|
||||
}
|
||||
|
||||
@Column(name = "address")
|
||||
public String getAddress() {
|
||||
return address;
|
||||
}
|
||||
|
||||
public void setAddress(String address) {
|
||||
this.address = address;
|
||||
}
|
||||
|
||||
@Column(name = "buildarea")
|
||||
public Double getBuildArea() {
|
||||
return buildArea;
|
||||
}
|
||||
|
||||
public void setBuildArea(Double buildArea) {
|
||||
this.buildArea = buildArea;
|
||||
}
|
||||
|
||||
@Column(name = "floor")
|
||||
public Integer getFloor() {
|
||||
return floor;
|
||||
}
|
||||
|
||||
public void setFloor(Integer floor) {
|
||||
this.floor = floor;
|
||||
}
|
||||
|
||||
@Column(name = "floorcnt")
|
||||
public Integer getFloorcnt() {
|
||||
return floorcnt;
|
||||
}
|
||||
|
||||
public void setFloorcnt(Integer floorcnt) {
|
||||
this.floorcnt = floorcnt;
|
||||
}
|
||||
|
||||
@Column(name = "roomcnt")
|
||||
public Integer getRoomcnt() {
|
||||
return roomcnt;
|
||||
}
|
||||
|
||||
public void setRoomcnt(Integer roomcnt) {
|
||||
this.roomcnt = roomcnt;
|
||||
}
|
||||
|
||||
@Column(name = "livingroomcnt")
|
||||
public Integer getLivingroomcnt() {
|
||||
return livingroomcnt;
|
||||
}
|
||||
|
||||
public void setLivingroomcnt(Integer livingroomcnt) {
|
||||
this.livingroomcnt = livingroomcnt;
|
||||
}
|
||||
|
||||
@Column(name = "houseaspect")
|
||||
public String getHouseaspect() {
|
||||
return houseaspect;
|
||||
}
|
||||
|
||||
public void setHouseaspect(String houseaspect) {
|
||||
this.houseaspect = houseaspect;
|
||||
}
|
||||
|
||||
@Column(name = "fitment")
|
||||
public String getFitment() {
|
||||
return fitment;
|
||||
}
|
||||
|
||||
public void setFitment(String fitment) {
|
||||
this.fitment = fitment;
|
||||
}
|
||||
|
||||
@Column(name = "c_date")
|
||||
public Date getcDate() {
|
||||
return cDate;
|
||||
}
|
||||
|
||||
public void setcDate(Date cDate) {
|
||||
this.cDate = cDate;
|
||||
}
|
||||
|
||||
@Column(name = "unitprice")
|
||||
public Integer getUnitPrice() {
|
||||
return unitPrice;
|
||||
}
|
||||
|
||||
public void setUnitPrice(Integer unitPrice) {
|
||||
this.unitPrice = unitPrice;
|
||||
}
|
||||
|
||||
@Column(name = "totalprice")
|
||||
public Integer getTotalPrice() {
|
||||
return totalPrice;
|
||||
}
|
||||
|
||||
public void setTotalPrice(Integer totalPrice) {
|
||||
this.totalPrice = totalPrice;
|
||||
}
|
||||
}
|
||||
@@ -63,6 +63,7 @@ public class DistrictFacilitys implements MongoEntity {
|
||||
private String imgUrl3D;
|
||||
|
||||
private List<String> school;
|
||||
private List<String> keySchool;
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@@ -380,4 +381,12 @@ public class DistrictFacilitys implements MongoEntity {
|
||||
this.school = school;
|
||||
}
|
||||
|
||||
@Column(name = "key_school")
|
||||
public List<String> getKeySchool() {
|
||||
return keySchool;
|
||||
}
|
||||
|
||||
public void setKeySchool(List<String> keySchool) {
|
||||
this.keySchool = keySchool;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,6 +48,11 @@ public class DealPrice implements MongoEntity {
|
||||
private Integer floor;//当前楼层
|
||||
private Long score;
|
||||
|
||||
private String matchId;
|
||||
private String matchCity;
|
||||
private String matchRegion;
|
||||
private String matchName;
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@Column(name = "_id", unique = true, nullable = false)
|
||||
@@ -296,4 +301,40 @@ public class DealPrice implements MongoEntity {
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Column(name = "match_id")
|
||||
public String getMatchId() {
|
||||
return matchId;
|
||||
}
|
||||
|
||||
public void setMatchId(String matchId) {
|
||||
this.matchId = matchId;
|
||||
}
|
||||
|
||||
@Column(name = "match_city")
|
||||
public String getMatchCity() {
|
||||
return matchCity;
|
||||
}
|
||||
|
||||
public void setMatchCity(String matchCity) {
|
||||
this.matchCity = matchCity;
|
||||
}
|
||||
|
||||
@Column(name = "match_region")
|
||||
public String getMatchRegion() {
|
||||
return matchRegion;
|
||||
}
|
||||
|
||||
public void setMatchRegion(String matchRegion) {
|
||||
this.matchRegion = matchRegion;
|
||||
}
|
||||
|
||||
@Column(name = "match_name")
|
||||
public String getMatchName() {
|
||||
return matchName;
|
||||
}
|
||||
|
||||
public void setMatchName(String matchName) {
|
||||
this.matchName = matchName;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -448,6 +448,7 @@ public class HouseLoudong extends com.fangjia.dal.mongo.model.HouseLoudong{
|
||||
this.openTotalPrice = openTotalPrice;
|
||||
}
|
||||
|
||||
@Column(name = "district_id")
|
||||
public String getDistrictId() {
|
||||
return districtId;
|
||||
}
|
||||
|
||||
@@ -101,7 +101,11 @@ public class Loudong implements MongoEntity {
|
||||
* seaweed中的小区名
|
||||
*/
|
||||
private String nameInSeaweed;
|
||||
|
||||
/**
|
||||
* seaweed中的小区id
|
||||
*/
|
||||
private String idInSeaweed;
|
||||
|
||||
@Column(name = "remark")
|
||||
public String getRemark() {
|
||||
return remark;
|
||||
@@ -402,4 +406,13 @@ public class Loudong implements MongoEntity {
|
||||
public void setNameInSeaweed(String nameInSeaweed) {
|
||||
this.nameInSeaweed = nameInSeaweed;
|
||||
}
|
||||
|
||||
@Column(name = "id_in_seaweed")
|
||||
public String getIdInSeaweed() {
|
||||
return idInSeaweed;
|
||||
}
|
||||
|
||||
public void setIdInSeaweed(String idInSeaweed) {
|
||||
this.idInSeaweed = idInSeaweed;
|
||||
}
|
||||
}
|
||||
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
package com.fangjia.dal.mongo.open.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
|
||||
public interface CommunityBasePriceDao extends MongoDao<CommunityBasePrice> {
|
||||
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
package com.fangjia.dal.mongo.open.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.model.CountDistrictStatBycity;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
|
||||
public interface CommunityPoiDao extends MongoDao<CommunityPoi> {
|
||||
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.fangjia.dal.mongo.open.dao.impl;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao;
|
||||
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
|
||||
public class CommunityBasePriceDaoImpl extends MongoDaoImpl<CommunityBasePrice> implements CommunityBasePriceDao {
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
package com.fangjia.dal.mongo.open.dao.impl;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.mongo.open.dao.ApiVisitCountDao;
|
||||
import com.fangjia.dal.mongo.open.dao.CommunityPoiDao;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
import com.fangjia.dal.mongo.tool.model.ApiVisitCount;
|
||||
|
||||
public class CommunityPoiDaoImpl extends MongoDaoImpl<CommunityPoi> implements CommunityPoiDao {
|
||||
}
|
||||
+74
@@ -0,0 +1,74 @@
|
||||
package com.fangjia.dal.mongo.open.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
|
||||
/**
|
||||
* Ð¡Çø½¨ÐÐÖÃÐŶÈÐÅÏ¢
|
||||
*
|
||||
* @author shenjian@fangjia.com
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "business.community_base_price_gj")
|
||||
public class CommunityBasePrice implements MongoEntity {
|
||||
|
||||
private static final long serialVersionUID = -8010010773207622878L;
|
||||
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String name;
|
||||
private Double credit;
|
||||
|
||||
@Id
|
||||
@Column(name = "_id")
|
||||
@Override
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "city")
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
|
||||
@Column(name = "region")
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
|
||||
public void setRegion(String region) {
|
||||
this.region = region;
|
||||
}
|
||||
|
||||
@Column(name = "name")
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
@Column(name = "credit")
|
||||
public Double getCredit() {
|
||||
return credit;
|
||||
}
|
||||
|
||||
public void setCredit(Double credit) {
|
||||
this.credit = credit;
|
||||
}
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
package com.fangjia.dal.mongo.open.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.*;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* 小区周边配套统计信息
|
||||
*
|
||||
* @author shenjian@fangjia.com
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "fangjia_stat.community_poi_gj")
|
||||
public class CommunityPoi implements MongoEntity {
|
||||
|
||||
private static final long serialVersionUID = -7663644539542318409L;
|
||||
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String name;
|
||||
private Integer subway;
|
||||
private Integer traffic;
|
||||
private Integer bank;
|
||||
private Integer primary;
|
||||
private Integer junior;
|
||||
private Integer hospital;
|
||||
/**
|
||||
* 不利设施,丧葬
|
||||
*/
|
||||
private Integer funeral;
|
||||
|
||||
@Id
|
||||
@Column(name = "_id")
|
||||
@Override
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "city")
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
|
||||
@Column(name = "region")
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
|
||||
public void setRegion(String region) {
|
||||
this.region = region;
|
||||
}
|
||||
|
||||
@Column(name = "name")
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
@Column(name = "subway")
|
||||
public Integer getSubway() {
|
||||
return subway;
|
||||
}
|
||||
|
||||
public void setSubway(Integer subway) {
|
||||
this.subway = subway;
|
||||
}
|
||||
|
||||
@Column(name = "traffic")
|
||||
public Integer getTraffic() {
|
||||
return traffic;
|
||||
}
|
||||
|
||||
public void setTraffic(Integer traffic) {
|
||||
this.traffic = traffic;
|
||||
}
|
||||
|
||||
@Column(name = "bank")
|
||||
public Integer getBank() {
|
||||
return bank;
|
||||
}
|
||||
|
||||
public void setBank(Integer bank) {
|
||||
this.bank = bank;
|
||||
}
|
||||
|
||||
@Column(name = "primary")
|
||||
public Integer getPrimary() {
|
||||
return primary;
|
||||
}
|
||||
|
||||
public void setPrimary(Integer primary) {
|
||||
this.primary = primary;
|
||||
}
|
||||
|
||||
@Column(name = "junior")
|
||||
public Integer getJunior() {
|
||||
return junior;
|
||||
}
|
||||
|
||||
public void setJunior(Integer junior) {
|
||||
this.junior = junior;
|
||||
}
|
||||
|
||||
@Column(name = "hospital")
|
||||
public Integer getHospital() {
|
||||
return hospital;
|
||||
}
|
||||
|
||||
public void setHospital(Integer hospital) {
|
||||
this.hospital = hospital;
|
||||
}
|
||||
|
||||
@Column(name = "funeral")
|
||||
public Integer getFuneral() {
|
||||
return funeral;
|
||||
}
|
||||
|
||||
public void setFuneral(Integer funeral) {
|
||||
this.funeral = funeral;
|
||||
}
|
||||
}
|
||||
+30
-5
@@ -67,6 +67,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
must.add(addTerms("district_name", alias));
|
||||
must.add(addMatch("cat", "sell"));
|
||||
must.add(addRange("avg_price", minPrice, maxPrice));
|
||||
must.add(addRangeGte("p_date", 20210501));
|
||||
|
||||
bool.put("must", must);
|
||||
query.put("bool", bool);
|
||||
reqBody.put("query", query);
|
||||
@@ -161,6 +163,16 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
return term;
|
||||
}
|
||||
|
||||
private JSONObject addRangeGte(String name, Number min){
|
||||
JSONObject term = new JSONObject();
|
||||
JSONObject range = new JSONObject();
|
||||
JSONObject rangeValue = new JSONObject();
|
||||
rangeValue.put("gte", min);
|
||||
range.put(name, rangeValue);
|
||||
term.put("range", range);
|
||||
return term;
|
||||
}
|
||||
|
||||
|
||||
private List<Seawater> wrapSeawater(List<JSONObject> jsonList){
|
||||
List<Seawater> seawaterList= new ArrayList<Seawater>();
|
||||
@@ -203,6 +215,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
seawater.setFloor(source.getInt("floor"));
|
||||
}catch(Exception e){
|
||||
seawater.setFloor(0);
|
||||
}
|
||||
try{
|
||||
seawater.setFloorInfo(source.getString("floor_info"));
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
if(source.containsKey("fitment")){
|
||||
seawater.setFitment(source.getString("fitment"));
|
||||
@@ -231,6 +248,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
if(source.containsKey("root")){
|
||||
seawater.setRoot(source.getString("root"));
|
||||
}
|
||||
|
||||
if(source.containsKey("estate_type")){
|
||||
seawater.setEstateType(source.getString("estate_type"));
|
||||
}
|
||||
|
||||
if(source.containsKey("build_type")){
|
||||
seawater.setBuildType(source.getString("build_type"));
|
||||
}
|
||||
seawaterList.add(seawater);
|
||||
}
|
||||
return seawaterList;
|
||||
@@ -379,11 +404,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
|
||||
}
|
||||
|
||||
// ²»Õ¹Ê¾É¾³ý
|
||||
JSONObject param = new JSONObject();
|
||||
param.put("status", 0);
|
||||
JSONObject temp = new JSONObject();
|
||||
temp.put("term", param);
|
||||
jsonList.add(temp);
|
||||
// JSONObject param = new JSONObject();
|
||||
// param.put("status", 0);
|
||||
// JSONObject temp = new JSONObject();
|
||||
// temp.put("term", param);
|
||||
// jsonList.add(temp);
|
||||
|
||||
query.put("must", jsonList);
|
||||
q.put("bool", query);
|
||||
|
||||
@@ -48,6 +48,7 @@ public class Seawater implements SeawaterEntity {
|
||||
private Double area;
|
||||
private Integer height;
|
||||
private Integer floor;
|
||||
private String floorInfo;
|
||||
private String fitment;
|
||||
private Integer room;
|
||||
private Integer hall;
|
||||
@@ -58,6 +59,9 @@ public class Seawater implements SeawaterEntity {
|
||||
private String url;
|
||||
private String root;
|
||||
private Integer avgPrice;
|
||||
private String purpose;
|
||||
private String estateType;
|
||||
private String buildType;
|
||||
|
||||
@Id
|
||||
@Column(name = "_id")
|
||||
@@ -195,6 +199,15 @@ public class Seawater implements SeawaterEntity {
|
||||
this.floor = floor;
|
||||
}
|
||||
|
||||
@Column(name = "floorInfo")
|
||||
public String getFloorInfo() {
|
||||
return floorInfo;
|
||||
}
|
||||
|
||||
public void setFloorInfo(String floorInfo) {
|
||||
this.floorInfo = floorInfo;
|
||||
}
|
||||
|
||||
@Column(name = "fitment")
|
||||
public String getFitment() {
|
||||
return fitment;
|
||||
@@ -285,4 +298,31 @@ public class Seawater implements SeawaterEntity {
|
||||
public void setAvgPrice(Integer avgPrice) {
|
||||
this.avgPrice = avgPrice;
|
||||
}
|
||||
|
||||
@Column(name = "purpose")
|
||||
public String getPurpose() {
|
||||
return purpose;
|
||||
}
|
||||
|
||||
public void setPurpose(String purpose) {
|
||||
this.purpose = purpose;
|
||||
}
|
||||
|
||||
@Column(name = "estate_type")
|
||||
public String getEstateType() {
|
||||
return estateType;
|
||||
}
|
||||
|
||||
public void setEstateType(String estateType) {
|
||||
this.estateType = estateType;
|
||||
}
|
||||
|
||||
@Column(name = "build_type")
|
||||
public String getBuildType() {
|
||||
return buildType;
|
||||
}
|
||||
|
||||
public void setBuildType(String buildType) {
|
||||
this.buildType = buildType;
|
||||
}
|
||||
}
|
||||
|
||||
+78
-2
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.open.controllers;
|
||||
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.SortParam;
|
||||
import com.fangjia.dal.common.DistrictKey;
|
||||
@@ -14,6 +15,7 @@ import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
import com.fangjia.service.fsh.DistrictNewPriceService;
|
||||
import com.fangjia.service.open.PaiMaiService;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.bson.types.ObjectId;
|
||||
|
||||
@@ -23,13 +25,14 @@ import java.util.regex.Pattern;
|
||||
/**
|
||||
* 法拍数据提供接口
|
||||
*/
|
||||
@Authority(role = Role.USER)
|
||||
//@Authority(role = Role.USER)
|
||||
public class AuctionController {
|
||||
|
||||
|
||||
private PaiMaiService paiMaiService;
|
||||
private DistrictNewPriceService districtNewPriceService;
|
||||
private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType"};
|
||||
private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url",
|
||||
"matchEstateType","type","count", "tradeAvgPrice", "evaluationPrice"};
|
||||
|
||||
public Object housesAction(AuctionModel model){
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
@@ -187,6 +190,79 @@ public class AuctionController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
public Object searchPaimaisAction(AuctionModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
if(StringUtils.isNotBlank(model.getCity())){
|
||||
params.add(new QueryParam("matchCity", model.getCity()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(model.getDistrict())){
|
||||
params.add(new QueryParam("matchRegion", model.getDistrict()));
|
||||
}
|
||||
|
||||
if(StringUtils.isNotBlank(model.getName())){
|
||||
params.add(new QueryParam("matchName", model.getName()));
|
||||
}
|
||||
|
||||
Date startDate = null;
|
||||
if(StringUtils.isNotBlank(model.getStartDate())){
|
||||
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
|
||||
if(startDate != null){
|
||||
params.add(new QueryParam("startDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
|
||||
}
|
||||
}
|
||||
|
||||
List<SortParam> sortParams = new ArrayList<SortParam>();
|
||||
sortParams.add(new SortParam("startDate", false));
|
||||
List<PaiMai> paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS);
|
||||
if(paimais!=null && paimais.size()>0){
|
||||
for(PaiMai paiMai : paimais){
|
||||
Map<String, Object> map = standard(paiMai.getCity(), paiMai.getAddress());
|
||||
if(map!=null){
|
||||
if (map.containsKey("buildingNumber")) {
|
||||
paiMai.setHouseNum((String) map.get("buildingNumber"));
|
||||
}
|
||||
if (map.containsKey("unitNumber")) {
|
||||
paiMai.setUnitNum((String) map.get("unitNumber"));
|
||||
}
|
||||
if (map.containsKey("roomNumber")) {
|
||||
paiMai.setRoomNum((String) map.get("roomNumber"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(paimais));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
private Map standard(String city, String address){
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", city);
|
||||
params.put("address", address);
|
||||
try{
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
|
||||
} catch(Exception e){
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8");
|
||||
} catch(Exception ex){
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("status");
|
||||
String msg = json.getString("msg");
|
||||
if(json.containsKey("result")){
|
||||
return (Map)json.get("result");
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public void setPaiMaiService(PaiMaiService paiMaiService) {
|
||||
this.paiMaiService = paiMaiService;
|
||||
}
|
||||
|
||||
+38
@@ -15,8 +15,10 @@ import java.util.Map.Entry;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.fangjia.dal.common.DistrictKey;
|
||||
import com.fangjia.dal.mongo.model.Metro;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.service.bo.DistrictStatBo;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
@@ -211,6 +213,42 @@ public class DealPriceController {
|
||||
mv.setModel(resMap);
|
||||
return mv;
|
||||
}
|
||||
|
||||
public Object searchListAction(final DealPriceModel model) {
|
||||
RestfulModelAndView mv = new RestfulModelAndView();
|
||||
mv.setViewType("rest");
|
||||
|
||||
if (StringUtils.isBlank(model.getId())) {
|
||||
mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空"));
|
||||
return mv;
|
||||
}
|
||||
District district = districtService.findDistrictById(model.getId());
|
||||
DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
|
||||
|
||||
if(district == null){
|
||||
mv.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
|
||||
return mv;
|
||||
}
|
||||
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
}
|
||||
Date startDate = null;
|
||||
Date endDate = null;
|
||||
if(StringUtils.isNotBlank(model.getStartDate())){
|
||||
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
|
||||
}
|
||||
if(StringUtils.isNotBlank(model.getEndDate())){
|
||||
endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd");
|
||||
}
|
||||
|
||||
List<DealPrice> dealPrices = dealPriceService.getNearlySameDealPrice(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit());
|
||||
mv.setModel(BasicModel.buildOKModel().setResult(dealPrices));
|
||||
return mv;
|
||||
}
|
||||
|
||||
private List<DealPrice> findDealPricesByName(String city,String region ,String name){
|
||||
Calendar calendar = Calendar.getInstance();
|
||||
|
||||
+37
-16
@@ -7,11 +7,11 @@ import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
import com.fangjia.dal.mongo.model.RegionStat;
|
||||
import com.fangjia.dal.mongo.model.RegionStatRent;
|
||||
import com.fangjia.dal.mongo.model.*;
|
||||
import com.fangjia.open.model.AvgPriceQueryModel;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.service.datastore.RegionStatService;
|
||||
import com.fangjia.service.district.DistrictFacilitysService;
|
||||
import com.fangjia.service.district.DistrictStatRentService;
|
||||
import com.fangjia.service.district.RegionStatRentService;
|
||||
import com.fangjia.service.mongo.fangjia.AverageAnalyService;
|
||||
@@ -38,8 +38,6 @@ import com.fangjia.dal.common.DistrictKey;
|
||||
import com.fangjia.dal.common.FangjiaServerAreaContextHolder;
|
||||
import com.fangjia.dal.common.ServerAreaData;
|
||||
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
|
||||
import com.fangjia.dal.mongo.model.DistrictHouse;
|
||||
import com.fangjia.dal.mongo.model.DistrictStat;
|
||||
import com.fangjia.dal.mongo.model.bkoff.ExtractDistrict;
|
||||
import com.fangjia.dal.seaweed.model.District;
|
||||
import com.fangjia.dal.sql.model.ApiUser;
|
||||
@@ -93,6 +91,7 @@ public class DistrictController {
|
||||
private RegionBlockService regionBlockService;
|
||||
private RegionStatService regionStatService;
|
||||
private RegionStatRentService regionStatRentService;
|
||||
private DistrictFacilitysService districtFacilitysService;
|
||||
|
||||
|
||||
public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea",
|
||||
@@ -444,19 +443,19 @@ public class DistrictController {
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
|
||||
@Param(name = "name") String name, HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
|
||||
return mav;
|
||||
} else {
|
||||
DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name));
|
||||
if (null == districtBo) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件"));
|
||||
public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
|
||||
@Param(name = "name") String name, HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
|
||||
return mav;
|
||||
}
|
||||
} else {
|
||||
DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name));
|
||||
if (null == districtBo) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件"));
|
||||
return mav;
|
||||
}
|
||||
districtBo.setStat(this.districtStatService.findDistrictStat(districtBo.getDistrictKey()));
|
||||
// 去除小区描述中的html标签
|
||||
districtBo.setDescription(DataUtil.removeSpecialString(districtBo.getDescription()));
|
||||
@@ -1146,6 +1145,25 @@ public class DistrictController {
|
||||
ret.put("RESULT", 200);
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object facilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region,
|
||||
@Param(name = "name") String name, HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入"));
|
||||
return mav;
|
||||
} else {
|
||||
District districtBo = districtService.findDistrict(new DistrictKey(city, region, name), "district");
|
||||
if (null == districtBo) {
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在"));
|
||||
return mav;
|
||||
}
|
||||
DistrictFacilitys facilitys = districtFacilitysService.findByDistrict(districtBo);
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(facilitys));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1230,4 +1248,7 @@ public class DistrictController {
|
||||
this.regionStatRentService = regionStatRentService;
|
||||
}
|
||||
|
||||
public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) {
|
||||
this.districtFacilitysService = districtFacilitysService;
|
||||
}
|
||||
}
|
||||
|
||||
+57
-5
@@ -10,9 +10,14 @@ import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.fangjia.dal.mongo.model.BsEvaluation;
|
||||
import com.fangjia.dal.mongo.model.bkoff.DealPrice;
|
||||
import com.fangjia.service.bo.AddrEvaResult;
|
||||
import com.fangjia.service.bo.DistrictStatBo;
|
||||
import com.fangjia.service.bo.EvaluationBo;
|
||||
import com.fangjia.service.bo.HouseScorer;
|
||||
import com.fangjia.service.datastore.*;
|
||||
import com.fangjia.service.district.DistrictStatService;
|
||||
import org.apache.commons.collections.MapUtils;
|
||||
import org.apache.commons.lang.ArrayUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
@@ -54,11 +59,6 @@ import com.fangjia.open.model.RestfulModelAndView;
|
||||
import com.fangjia.service.api.AutoCompleteService;
|
||||
import com.fangjia.service.api.KnowbaseService;
|
||||
import com.fangjia.service.common.SellSimilar;
|
||||
import com.fangjia.service.datastore.BusinessEstateService;
|
||||
import com.fangjia.service.datastore.DistrictService;
|
||||
import com.fangjia.service.datastore.EvaluationReportService;
|
||||
import com.fangjia.service.datastore.EvaluationService;
|
||||
import com.fangjia.service.datastore.HouseService;
|
||||
import com.fangjia.service.sql.ApiUserService;
|
||||
import com.fangjia.service.sql.UserService;
|
||||
import com.fangjia.service.web.AddressService;
|
||||
@@ -94,6 +94,8 @@ public class EvaluationController {
|
||||
private ApiUserService apiUserService;
|
||||
private UserService userService;
|
||||
private AddressService addressService;
|
||||
private DistrictStatService districtStatService;
|
||||
private BsEvaluationService bsEvaluationService;
|
||||
|
||||
private final static String AK = OpenConfig.getConfig().getBaiduAk();
|
||||
public Object lngLatEvaluateAction(EvaluationModel model) {
|
||||
@@ -1005,6 +1007,44 @@ public class EvaluationController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object bsListAction(EvaluationModel model) {
|
||||
RestfulModelAndView mv = new RestfulModelAndView();
|
||||
mv.setViewType("rest");
|
||||
|
||||
if (StringUtils.isBlank(model.getId())) {
|
||||
mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空"));
|
||||
return mv;
|
||||
}
|
||||
District district = districtService.findDistrictById(model.getId());
|
||||
DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
|
||||
|
||||
if(district == null){
|
||||
mv.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
|
||||
return mv;
|
||||
}
|
||||
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
}
|
||||
Date startDate = null;
|
||||
Date endDate = null;
|
||||
if(StringUtils.isNotBlank(model.getStartDate())){
|
||||
startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd");
|
||||
}
|
||||
if(StringUtils.isNotBlank(model.getEndDate())){
|
||||
endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd");
|
||||
}
|
||||
|
||||
List<BsEvaluation> dealPrices = bsEvaluationService.getNearlySameEvaluation(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit());
|
||||
mv.setModel(BasicModel.buildOKModel().setResult(dealPrices));
|
||||
return mv;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void setEvaluationService(EvaluationService evaluationService) {
|
||||
this.evaluationService = evaluationService;
|
||||
}
|
||||
@@ -1017,6 +1057,18 @@ public class EvaluationController {
|
||||
this.knowbaseService = knowbaseService;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void setDistrictStatService(DistrictStatService districtStatService) {
|
||||
this.districtStatService = districtStatService;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void setBsEvaluationService(BsEvaluationService bsEvaluationService) {
|
||||
this.bsEvaluationService = bsEvaluationService;
|
||||
}
|
||||
|
||||
public void setAutoCompleteService(AutoCompleteService autoCompleteService) {
|
||||
this.autoCompleteService = autoCompleteService;
|
||||
}
|
||||
|
||||
@@ -11,6 +11,8 @@ import com.fangjia.dal.mongo.model.DistrictFacilitys;
|
||||
import com.fangjia.dal.mongo.model.FangjiaOfficialAreaMap;
|
||||
import com.fangjia.dal.mongo.model.Metro;
|
||||
import com.fangjia.dal.mongo.model.StatsIndex;
|
||||
import com.fangjia.dal.seawater.dao.SeawaterDao;
|
||||
import com.fangjia.dal.seawater.model.Seawater;
|
||||
import com.fangjia.dal.seaweed.model.District;
|
||||
import com.fangjia.dal.sql.model.ApiUser;
|
||||
import com.fangjia.datastore.driver.FacetResult;
|
||||
@@ -29,6 +31,7 @@ import com.fangjia.open.ws.query.CollateralSampleQuery;
|
||||
import com.fangjia.open.ws.query.ShopCollateralSampleQuery;
|
||||
import com.fangjia.service.address.AddressParser;
|
||||
import com.fangjia.service.bo.Collateral;
|
||||
import com.fangjia.service.bo.DistrictStatBo;
|
||||
import com.fangjia.service.bo.Sample;
|
||||
import com.fangjia.service.bo.ShopSample;
|
||||
import com.fangjia.service.datastore.CzbankSampleService;
|
||||
@@ -39,11 +42,13 @@ import com.fangjia.service.datastore.bean.CollateralSampleQueryBean;
|
||||
import com.fangjia.service.datastore.bean.ShopCollateralSampleQueryBean;
|
||||
import com.fangjia.service.district.DistrictFacilitysService;
|
||||
import com.fangjia.service.district.DistrictStatService;
|
||||
import com.fangjia.service.district.SqmetreRentService;
|
||||
import com.fangjia.service.menu.Item;
|
||||
import com.fangjia.service.menu.MenuCategoryEnum;
|
||||
import com.fangjia.service.menu.Value;
|
||||
import com.fangjia.service.mongo.fangjia.DealPriceService;
|
||||
import com.fangjia.service.mongo.fangjia.MetroService;
|
||||
import com.fangjia.service.mongo.fangjia.SeawaterService;
|
||||
import com.fangjia.service.mongo.fangjia.StatsIndexService;
|
||||
import com.fangjia.service.open.FangjiaOfficialAreaService;
|
||||
import com.fangjia.service.open.HousePojo;
|
||||
@@ -86,6 +91,9 @@ public class HouseController {
|
||||
private CzbankSampleService czbankSampleService;
|
||||
private DistrictService districtService;
|
||||
private FangjiaOfficialAreaService fangjiaOfficialAreaService;
|
||||
|
||||
private SeawaterService seawaterService;
|
||||
private SeawaterDao seawaterDao;
|
||||
private List<String> PROPERTY_TYPE_CODE_LIST = Arrays.asList("01", "02");
|
||||
|
||||
|
||||
@@ -399,6 +407,51 @@ public class HouseController {
|
||||
}
|
||||
}
|
||||
|
||||
public Object listAction(final HouseSearchModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项"));
|
||||
return mav;
|
||||
}
|
||||
District district = null;
|
||||
DistrictKey districtKey = null;
|
||||
if(StringUtils.isNotBlank(model.getName())){
|
||||
districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName());
|
||||
district = districtService.findDistrict(districtKey, "district");
|
||||
}else{
|
||||
district = districtService.findDistrictById(model.getId());
|
||||
if(district!=null){
|
||||
districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName());
|
||||
}
|
||||
}
|
||||
|
||||
if(district == null){
|
||||
mav.setModel(BasicModel.buildBadRequestModel("该小区不存在"));
|
||||
return mav;
|
||||
}
|
||||
DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey);
|
||||
Integer minPrice = null;
|
||||
Integer maxPrice = null;
|
||||
if(bo.getDistrictStat()!=null){
|
||||
minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8);
|
||||
maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2);
|
||||
}
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add(district.getName());
|
||||
if(district.getAlias() != null){
|
||||
for(String alia : district.getAlias()){
|
||||
if(StringUtils.isNotBlank(alia) && !names.contains(alia)){
|
||||
names.add(alia);
|
||||
}
|
||||
}
|
||||
}
|
||||
List<Seawater> houses = seawaterDao.findSeawatersFromEs("sell", district.getCity(), district.getRegion(), district.getName(), district.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater");
|
||||
//List<Seawater> houses1 = seawaterDao.findHouses(district.getCity(), district.getRegion(), names, minPrice, maxPrice, model.getPageSize());
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(houses));
|
||||
return mav;
|
||||
}
|
||||
|
||||
String getPicUrl(String url, List<String> urls) {
|
||||
if (!urls.contains(url)) {
|
||||
urls.add(url);
|
||||
@@ -832,6 +885,13 @@ public class HouseController {
|
||||
return ret;
|
||||
}
|
||||
|
||||
public void setSeawaterService(SeawaterService seawaterService) {
|
||||
this.seawaterService = seawaterService;
|
||||
}
|
||||
|
||||
public void setSeawaterDao(SeawaterDao seawaterDao) {
|
||||
this.seawaterDao = seawaterDao;
|
||||
}
|
||||
|
||||
private boolean checkExists(FacetQueryResult<HousePojo> houseBoList, HousePojo house) {
|
||||
for (HousePojo bo : houseBoList) {
|
||||
|
||||
+220
-53
@@ -31,12 +31,15 @@ import javax.validation.Valid;
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.dal.*;
|
||||
import com.fangjia.dal.mongo.model.*;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
import com.fangjia.dal.seawater.dao.SeawaterDao;
|
||||
import com.fangjia.dal.seawater.model.Seawater;
|
||||
import com.fangjia.open.dto.*;
|
||||
import com.fangjia.service.district.*;
|
||||
import com.fangjia.service.mongo.fangjia.*;
|
||||
import com.fangjia.service.open.*;
|
||||
import com.fangjia.service.open.impl.CommunityPoiServiceImpl;
|
||||
import com.fangjia.service.util.DistrictScoreUtil;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONObject;
|
||||
@@ -175,6 +178,9 @@ public class PropertyController {
|
||||
private SeawaterDao seawaterDao;
|
||||
private SeawaterService seawaterService;
|
||||
private SqmetreRentService sqmetreRentService;
|
||||
private CommunityPoiService communityPoiService;
|
||||
private CommunityBasePriceService communityBasePriceService;
|
||||
private EvaluateFactorService evaluateFactorService;
|
||||
|
||||
private FileDao fileDao = FileDao.getInstance();
|
||||
private static final int DEFAULT_PAGE_SIZE = 20;
|
||||
@@ -390,10 +396,31 @@ public class PropertyController {
|
||||
put("5000", "five");// 五线
|
||||
}
|
||||
};
|
||||
/**
|
||||
* 第三方均价映射
|
||||
*/
|
||||
private static final Map<String, String> OTHER_PRICE_MAP = new HashMap<String, String>() {
|
||||
{
|
||||
put("zhuge_price", "诸葛找房");
|
||||
put("anjuke_price", "安居客");
|
||||
put("fang_price", "房天下");
|
||||
put("lianjia_price", "链家");
|
||||
put("cityhouse_price", "禧泰数据");
|
||||
put("maitian_price", "麦田");
|
||||
put("ke_price", "贝壳");
|
||||
put("taofang_price", "淘房365");
|
||||
put("zhongyuan_price", "中原");
|
||||
put("tbs_price", "兔博士");
|
||||
put("qfang_price", "Q房");
|
||||
put("fangtan_price", "房探");
|
||||
put("fangduoduo_price", "房多多");
|
||||
put("wiwj_price", "我爱我家");
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 利用百度API把GPS转换成Google经纬度
|
||||
*
|
||||
*
|
||||
* @param lng
|
||||
* @param lat
|
||||
* @return
|
||||
@@ -415,7 +442,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 利用算法模板把GPS转换成Google经纬度
|
||||
*
|
||||
*
|
||||
* @param lng
|
||||
* @param lat
|
||||
* @return
|
||||
@@ -607,7 +634,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 判断小区是否有经纬度
|
||||
*
|
||||
*
|
||||
* @param district
|
||||
* @return
|
||||
*/
|
||||
@@ -618,7 +645,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 调用匹配接口匹配小区
|
||||
*
|
||||
*
|
||||
* @param city
|
||||
* @param address
|
||||
* @param region
|
||||
@@ -666,17 +693,6 @@ public class PropertyController {
|
||||
}
|
||||
String userId = FangjiaApiUtil.parseUserId(model.getToken());
|
||||
|
||||
/**超级用户和上海交大账户不控制权限*/
|
||||
if (!"1007497".equals(userId) && !"1007632".equals(userId) && !"1007548".equals(userId)
|
||||
&& !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId)
|
||||
&& !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId)
|
||||
&& !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId)
|
||||
&& !"1007747".equals(userId) && !"1007757".equals(userId)
|
||||
&& model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel());
|
||||
return mav;
|
||||
}
|
||||
|
||||
District district = null;
|
||||
if (StringUtils.isNotBlank(model.getId())){
|
||||
if(ObjectId.isValid(model.getId())){
|
||||
@@ -740,18 +756,18 @@ public class PropertyController {
|
||||
} else {
|
||||
districtInfoDto.setBuildingType("");
|
||||
}
|
||||
districtInfoDto.setAlias(district.getAlias());
|
||||
districtInfoDto.setAlias(district.getMappingAlias());
|
||||
districtInfoDto.setAddressAlias(district.getAddrAlias());
|
||||
districtInfoDto.setCommunity(district.getStreetCommunity());
|
||||
districtInfoDto.setCommercialArea(district.getBusinessQuarter());
|
||||
if (district.getHouseholdCount() != null) {
|
||||
districtInfoDto.setTotalUnitCount(district.getHouseholdCount().toString());
|
||||
districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString()));
|
||||
} else {
|
||||
districtInfoDto.setTotalUnitCount("");
|
||||
}
|
||||
districtInfoDto.setManagementFee(district.getEstateCharge());
|
||||
if (district.getCarSiteCount() != null) {
|
||||
districtInfoDto.setParking(district.getCarSiteCount().toString());
|
||||
districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString()));
|
||||
} else {
|
||||
districtInfoDto.setParking("");
|
||||
}
|
||||
@@ -772,10 +788,10 @@ public class PropertyController {
|
||||
|
||||
districtInfoDto.setFinishBuildTime(district.getFinishBuildingDate());
|
||||
districtInfoDto.setMoveinTime(district.getLiveDate());
|
||||
districtInfoDto.setFloorSpace(district.getAllArea());
|
||||
districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea()));
|
||||
districtInfoDto.setConstructionSpace(district.getBuildArea());
|
||||
districtInfoDto.setPlotRatio(district.getCubagePercent());
|
||||
districtInfoDto.setGreenPercentage(district.getVirescencePercent());
|
||||
districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent()));
|
||||
districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent()));
|
||||
districtInfoDto.setPreSalePermit(district.getLicence());
|
||||
districtInfoDto.setDeveloper(district.getDeveloper());
|
||||
districtInfoDto.setDiscount(district.getDiscount());
|
||||
@@ -792,6 +808,7 @@ public class PropertyController {
|
||||
districtInfoDto.setDescription(district.getDescription());
|
||||
districtInfoDto.setSalesOfficeAdd(district.getSellAddress());
|
||||
districtInfoDto.setSalesPhoneNumber(district.getMobile());
|
||||
districtInfoDto.setTownship(district.getTownship());
|
||||
if (district.getLoudongCount() != null && district.getLoudongCount() > 0) {
|
||||
districtInfoDto.setBuildingCount(district.getLoudongCount());
|
||||
}
|
||||
@@ -960,6 +977,12 @@ public class PropertyController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
|
||||
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
|
||||
if(loudong != null && loudong.getFloors() >= model.getFloor()){
|
||||
model.setMaxFloor(loudong.getFloors());
|
||||
}
|
||||
}
|
||||
model.setRegion(district.getRegion());
|
||||
model.setDistrictName(district.getName());
|
||||
if (StringUtils.isBlank(model.getPropertyType())) {
|
||||
@@ -996,6 +1019,32 @@ public class PropertyController {
|
||||
}
|
||||
}
|
||||
|
||||
// 调整面积不同面积价格
|
||||
try {
|
||||
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
|
||||
int orgAvgPrice = 0;
|
||||
if (districtStatBo != null) {
|
||||
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
|
||||
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
|
||||
} else if (districtStatBo.getDistrictStat() != null) {
|
||||
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
|
||||
}
|
||||
}
|
||||
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
|
||||
int factorPrice = evaluation.getAvgPrice();
|
||||
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
|
||||
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
|
||||
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
|
||||
// 偏离度超过5%不进行调整
|
||||
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||
evaluation.setAvgPrice(avgPrice);
|
||||
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
|
||||
}
|
||||
|
||||
if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0
|
||||
&& (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) {
|
||||
Map<String, Object> districtPriceMap = new HashMap<String, Object>();
|
||||
@@ -1116,7 +1165,7 @@ public class PropertyController {
|
||||
model.setCalDate(calDate);
|
||||
}
|
||||
/* 先到CPDB库中查找小区 */
|
||||
District district = matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName());
|
||||
District district = districtService.findDistrict(new DistrictKey(model.getCity(), model.getDistrict(), model.getName()), "district");//matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName());
|
||||
if(district == null){
|
||||
//防止地址模糊,名称详细的情况
|
||||
district = matchDistrict(model.getCity(), model.getName(), model.getDistrict(), model.getAddress());
|
||||
@@ -1166,19 +1215,52 @@ public class PropertyController {
|
||||
evaluationService.remove(evaluation.getId());
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
|
||||
return mav;
|
||||
}
|
||||
// 调整面积不同面积价格
|
||||
try {
|
||||
DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district));
|
||||
int orgAvgPrice = 0;
|
||||
if (districtStatBo != null) {
|
||||
if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) {
|
||||
orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice();
|
||||
} else if (districtStatBo.getDistrictStat() != null) {
|
||||
orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice();
|
||||
}
|
||||
}
|
||||
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
|
||||
int factorPrice = evaluation.getAvgPrice();
|
||||
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
|
||||
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95);
|
||||
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05);
|
||||
// 偏离度超过5%不进行调整
|
||||
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||
evaluation.setAvgPrice(avgPrice);
|
||||
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
|
||||
}
|
||||
Map<String, Object> districtPriceMap = new LinkedHashMap<String, Object>();
|
||||
districtPriceMap.put("city", district.getCity());
|
||||
districtPriceMap.put("district", district.getRegion());
|
||||
districtPriceMap.put("name", district.getName());
|
||||
long totalPrice = evaluation.getBigTotalPrice();
|
||||
if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
|
||||
totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
|
||||
evaluation.setBigTotalPrice(totalPrice);
|
||||
evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
|
||||
}
|
||||
// 取消系数下调
|
||||
// if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){
|
||||
// totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity())));
|
||||
// evaluation.setBigTotalPrice(totalPrice);
|
||||
// evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea()));
|
||||
// }
|
||||
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
|
||||
districtPriceMap.put("unitPrice", evaluation.getAvgPrice());
|
||||
// 新增置信度
|
||||
CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId());
|
||||
if (communityBasePrice != null) {
|
||||
districtPriceMap.put("credit", communityBasePrice.getCredit());
|
||||
} else {
|
||||
districtPriceMap.put("credit", evaluation.getCredit());
|
||||
}
|
||||
List<String> names = new ArrayList<String>();
|
||||
names.add(district.getName());
|
||||
if(district.getAlias() != null){
|
||||
@@ -1190,6 +1272,31 @@ public class PropertyController {
|
||||
}
|
||||
Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8);
|
||||
Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2);
|
||||
Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9);
|
||||
Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1);
|
||||
// 新增第三方均价
|
||||
try {
|
||||
double dCredit = 0.61;
|
||||
Map<String, Object> rMap = getLastEvaluationPrice(district, evaluation, dCredit);
|
||||
if (null != rMap) {
|
||||
if (rMap.containsKey("other_price")) {
|
||||
Map<String, Integer> otherPriceMap = (Map<String, Integer>) rMap.get("other_price");
|
||||
Map<String, Integer> priceResultMap = new HashMap<String, Integer>();
|
||||
for (String key : otherPriceMap.keySet()) {
|
||||
Integer price = otherPriceMap.get(key);
|
||||
// 剔除偏差大于10的
|
||||
if (min10Price <= price && max10Price >= price) {
|
||||
if (OTHER_PRICE_MAP.containsKey(key)) {
|
||||
priceResultMap.put(OTHER_PRICE_MAP.get(key), price);
|
||||
}
|
||||
}
|
||||
}
|
||||
districtPriceMap.put("otherPrice", priceResultMap);
|
||||
}
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
|
||||
}
|
||||
List<Seawater> houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20);
|
||||
if(houses.size() > 3){
|
||||
districtPriceMap.put("calcType", 1);
|
||||
@@ -1218,7 +1325,7 @@ public class PropertyController {
|
||||
}else if(district.getLng() != null && district.getLng() > 0){
|
||||
double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6);
|
||||
double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6);
|
||||
List<District> districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 4, null);
|
||||
List<District> districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 6, null);
|
||||
if(districts.size() < 2){
|
||||
evaluationService.remove(evaluation.getId());
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData());
|
||||
@@ -1256,9 +1363,15 @@ public class PropertyController {
|
||||
districtDto.append("buildYear", String.valueOf(buildYear));
|
||||
}
|
||||
districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d));
|
||||
if(districtAnalyData != null && districtAnalyData.getHalfYearMarkup() != null){
|
||||
districtDto.append("halfYearMarkup",DataUtil.roundDouble( districtAnalyData.getHalfYearMarkup() * 100 ,2));
|
||||
if(districtAnalyData != null){
|
||||
if (districtAnalyData.getHalfYearMarkup() != null) {
|
||||
districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2));
|
||||
}
|
||||
if (districtAnalyData.getAvgPrice() != null && minPrice <= districtAnalyData.getAvgPrice() && maxPrice >= districtAnalyData.getAvgPrice()) {
|
||||
districtDto.append("avgPrice", districtAnalyData.getAvgPrice());
|
||||
}
|
||||
}
|
||||
|
||||
nearCommunityDtos.add(districtDto);
|
||||
}
|
||||
Integer buildYear = districtService.getBuildYear(district);
|
||||
@@ -1266,6 +1379,18 @@ public class PropertyController {
|
||||
modelFeaturesFto.put("buildYear", String.valueOf(buildYear));
|
||||
}
|
||||
modelFeaturesFto.append("nearCommunity", nearCommunityDtos);
|
||||
|
||||
// 模型特征新增周边数量
|
||||
CommunityPoi communityPoi = communityPoiService.getById(district.getId());
|
||||
if (communityPoi != null) {
|
||||
modelFeaturesFto.put("subway", communityPoi.getSubway());
|
||||
modelFeaturesFto.put("traffic", communityPoi.getTraffic());
|
||||
modelFeaturesFto.put("bank", communityPoi.getBank());
|
||||
modelFeaturesFto.put("primary", communityPoi.getPrimary());
|
||||
modelFeaturesFto.put("hospital", communityPoi.getHospital());
|
||||
modelFeaturesFto.put("junior", communityPoi.getJunior());
|
||||
modelFeaturesFto.put("funeral", communityPoi.getFuneral());
|
||||
}
|
||||
districtPriceMap.put("modelFeatures", modelFeaturesFto);
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap));
|
||||
@@ -1473,7 +1598,7 @@ public class PropertyController {
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 四川锦程 定制接口 多建筑年代
|
||||
* @param model
|
||||
@@ -1585,7 +1710,7 @@ public class PropertyController {
|
||||
districtPriceMap.put("buildYear", matcher3.group());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// 根据城市等 级设置D级可信度
|
||||
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
|
||||
@@ -2098,7 +2223,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区租金查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -2577,7 +2702,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 批量获取小区的均价,均价走势数据
|
||||
*
|
||||
*
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
@@ -2719,7 +2844,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 批量获取小区的配套数据,最多12个小区
|
||||
*
|
||||
*
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
@@ -2864,7 +2989,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 获取JSON参数中的num参数值
|
||||
*
|
||||
*
|
||||
* @param jo
|
||||
* @return
|
||||
*/
|
||||
@@ -2889,7 +3014,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 获取JSON参数中的radius值
|
||||
*
|
||||
*
|
||||
* @param jo
|
||||
* @return
|
||||
*/
|
||||
@@ -3697,7 +3822,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 解析出关键字中的小区名
|
||||
*
|
||||
*
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
@@ -3814,7 +3939,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区统计分析接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -3899,7 +4024,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区交通接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -3957,7 +4082,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区周边接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4016,7 +4141,7 @@ public class PropertyController {
|
||||
/**
|
||||
* 相同小区相似户型房源挂牌量接口<br>
|
||||
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
|
||||
*
|
||||
*
|
||||
* @author yinjihuan
|
||||
* @date 2016年9月1日
|
||||
* @param city
|
||||
@@ -4056,7 +4181,7 @@ public class PropertyController {
|
||||
/**
|
||||
* 周边小区相似户型挂牌量接口<br>
|
||||
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源
|
||||
*
|
||||
*
|
||||
* @author yinjihuan
|
||||
* @date 2016年9月1日
|
||||
* @param city
|
||||
@@ -4107,7 +4232,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 小区周边一公里均价
|
||||
*
|
||||
*
|
||||
* @author wenyan
|
||||
* @param model
|
||||
* @return
|
||||
@@ -4130,7 +4255,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 相同小区相似户型租金范围查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4173,7 +4298,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 周边小区相似户型租金范围查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4236,7 +4361,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 室号查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4272,7 +4397,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 楼栋楼层信息查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4297,7 +4422,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 楼栋查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4342,7 +4467,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 楼栋查询接口
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4795,7 +4920,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 房产交易记录查询
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4855,7 +4980,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 房产拍卖记录
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -4916,7 +5041,7 @@ public class PropertyController {
|
||||
|
||||
/**
|
||||
* 房产近期评估次数
|
||||
*
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
@@ -5412,6 +5537,36 @@ public class PropertyController {
|
||||
}
|
||||
}
|
||||
|
||||
public Object searchHouseLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
|
||||
if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber()) || StringUtils.isBlank(model.getRoomNumber())){
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号,室为必填项"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
List<HouseLoudong> houseLoudongs = houseLoudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber(), model.getRoomNumber());
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(houseLoudongs));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
public Object searchLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
|
||||
if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber())){
|
||||
mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号为必填项"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
List<Loudong> loudongs = loudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber());
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(loudongs));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
public void setHouseLoudongService(HouseLoudongService houseLoudongService) {
|
||||
this.houseLoudongService = houseLoudongService;
|
||||
@@ -5601,4 +5756,16 @@ public class PropertyController {
|
||||
public void setRentDealPriceService(RentDealPriceService rentDealPriceService) {
|
||||
this.rentDealPriceService = rentDealPriceService;
|
||||
}
|
||||
|
||||
public void setCommunityPoiService(CommunityPoiService communityPoiService) {
|
||||
this.communityPoiService = communityPoiService;
|
||||
}
|
||||
|
||||
public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) {
|
||||
this.evaluateFactorService = evaluateFactorService;
|
||||
}
|
||||
|
||||
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
|
||||
this.communityBasePriceService = communityBasePriceService;
|
||||
}
|
||||
}
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package com.fangjia.open.controllers;
|
||||
|
||||
import com.fangjia.core.util.AnalyticsUtil;
|
||||
import com.fangjia.open.interceptors.Authority;
|
||||
import com.fangjia.open.interceptors.Role;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import java.util.Enumeration;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
public class requestController {
|
||||
|
||||
/**
|
||||
* »ñÈ¡ÇëÇóµÄrequestºÍIPÐÅÏ¢
|
||||
* @param request
|
||||
* @return
|
||||
*/
|
||||
public Object requestAction(HttpServletRequest request) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
String ip = AnalyticsUtil.getIpAddress(request);
|
||||
Map<String, String> map = new HashMap<String, String>();
|
||||
Enumeration headerNames = request.getHeaderNames();
|
||||
while (headerNames.hasMoreElements()) {
|
||||
String key = (String) headerNames.nextElement();
|
||||
String value = request.getHeader(key);
|
||||
map.put(key, value);
|
||||
}
|
||||
map.put("ip", ip);
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(map));
|
||||
return mav;
|
||||
}
|
||||
}
|
||||
@@ -99,6 +99,7 @@ public class DistrictInfoDto {
|
||||
private String salesPhoneNumber;
|
||||
private Integer buildingCount;
|
||||
private Integer finishYear;
|
||||
private String township;
|
||||
|
||||
|
||||
public String getDistrict() {
|
||||
@@ -818,4 +819,12 @@ public class DistrictInfoDto {
|
||||
public void setFinishYear(Integer finishYear) {
|
||||
this.finishYear = finishYear;
|
||||
}
|
||||
|
||||
public String getTownship() {
|
||||
return township;
|
||||
}
|
||||
|
||||
public void setTownship(String township) {
|
||||
this.township = township;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,8 +75,8 @@ public class AnalyticsFilter implements Filter {
|
||||
if ("1".equals(request.getParameter("__refresh_cache"))) {
|
||||
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
|
||||
}
|
||||
createUID(request, response);
|
||||
createSID(request, response);
|
||||
//createUID(request, response);
|
||||
//createSID(request, response);
|
||||
response.addHeader("Access-Control-Allow-Origin", "*");
|
||||
response.addHeader("Access-Control-Allow-Methods", "*");
|
||||
chain.doFilter(request, response);
|
||||
|
||||
@@ -8,6 +8,7 @@ public class AuctionModel extends FangjiaApiModel{
|
||||
private String city;
|
||||
private String district;
|
||||
private String address;
|
||||
private String name;
|
||||
private Date minStartTime;
|
||||
private Date maxStartTime;
|
||||
private Integer minStartPrice;
|
||||
@@ -17,6 +18,7 @@ public class AuctionModel extends FangjiaApiModel{
|
||||
private Integer propertyType;
|
||||
private Integer status;
|
||||
private Integer limit = 10;
|
||||
private String startDate;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
@@ -121,4 +123,20 @@ public class AuctionModel extends FangjiaApiModel{
|
||||
public void setStatus(Integer status) {
|
||||
this.status = status;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getStartDate() {
|
||||
return startDate;
|
||||
}
|
||||
|
||||
public void setStartDate(String startDate) {
|
||||
this.startDate = startDate;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package com.fangjia.open.model;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class DealPriceModel extends FangjiaApiModel{
|
||||
private String id;
|
||||
|
||||
private String city;
|
||||
|
||||
@@ -34,9 +37,13 @@ public class DealPriceModel extends FangjiaApiModel{
|
||||
private Double size;
|
||||
|
||||
private Integer max;
|
||||
private Integer limit = 10;
|
||||
private String decoration;
|
||||
|
||||
private boolean convert;
|
||||
|
||||
private String startDate;
|
||||
private String endDate;
|
||||
|
||||
|
||||
public String getCity() {
|
||||
@@ -198,4 +205,36 @@ public class DealPriceModel extends FangjiaApiModel{
|
||||
public void setConvert(boolean convert) {
|
||||
this.convert = convert;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getStartDate() {
|
||||
return startDate;
|
||||
}
|
||||
|
||||
public void setStartDate(String startDate) {
|
||||
this.startDate = startDate;
|
||||
}
|
||||
|
||||
public String getEndDate() {
|
||||
return endDate;
|
||||
}
|
||||
|
||||
public void setEndDate(String endDate) {
|
||||
this.endDate = endDate;
|
||||
}
|
||||
|
||||
public Integer getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
public void setLimit(Integer limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -246,6 +246,42 @@ public class EvaluationModel extends FangjiaApiModel {
|
||||
private Date buildDate;
|
||||
private String sid;//微信小程序中的sid
|
||||
private String code;//微信小程序中的code
|
||||
private String id;
|
||||
private String startDate;
|
||||
private String endDate;
|
||||
private Integer limit = 10;
|
||||
|
||||
public Integer getLimit() {
|
||||
return limit;
|
||||
}
|
||||
|
||||
public void setLimit(Integer limit) {
|
||||
this.limit = limit;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getStartDate() {
|
||||
return startDate;
|
||||
}
|
||||
|
||||
public void setStartDate(String startDate) {
|
||||
this.startDate = startDate;
|
||||
}
|
||||
|
||||
public String getEndDate() {
|
||||
return endDate;
|
||||
}
|
||||
|
||||
public void setEndDate(String endDate) {
|
||||
this.endDate = endDate;
|
||||
}
|
||||
|
||||
public String getRegion() {
|
||||
return region;
|
||||
|
||||
@@ -4,6 +4,7 @@ import com.fangjia.service.datastore.bean.HouseQueryBean;
|
||||
import com.fangjia.service.query.QueryBean;
|
||||
|
||||
public class HouseSearchModel extends SearchModel {
|
||||
private String id;
|
||||
private String kind;
|
||||
private String estateType;
|
||||
private String buildingType;
|
||||
@@ -116,7 +117,12 @@ public class HouseSearchModel extends SearchModel {
|
||||
public void setShowDetail(Boolean showDetail) {
|
||||
this.showDetail = showDetail;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
|
||||
+21
@@ -46,7 +46,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
|
||||
* Â¥ºÅ
|
||||
*/
|
||||
private String unitNumber;
|
||||
/**
|
||||
* ÐÂÂ¥ºÅ
|
||||
*/
|
||||
private String houseNumber;
|
||||
|
||||
private String id;
|
||||
/**
|
||||
* ÊÒºÅ
|
||||
*/
|
||||
@@ -905,4 +910,20 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
|
||||
public void setAvgPrice(Integer avgPrice) {
|
||||
this.avgPrice = avgPrice;
|
||||
}
|
||||
|
||||
public String getHouseNumber() {
|
||||
return houseNumber;
|
||||
}
|
||||
|
||||
public void setHouseNumber(String houseNumber) {
|
||||
this.houseNumber = houseNumber;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,8 @@ public abstract class SearchModel extends FangjiaApiModel {
|
||||
protected Integer totalPriceEnd;
|
||||
protected Integer avgPriceStart;
|
||||
protected Integer avgPriceEnd;
|
||||
protected Integer startTime;
|
||||
protected Integer endTime;
|
||||
protected Integer areaStart;
|
||||
protected Integer areaEnd;
|
||||
protected Integer room;
|
||||
@@ -358,4 +360,20 @@ public abstract class SearchModel extends FangjiaApiModel {
|
||||
public void setTotalPrice(Integer totalPrice) {
|
||||
this.totalPrice = totalPrice;
|
||||
}
|
||||
|
||||
public Integer getStartTime() {
|
||||
return startTime;
|
||||
}
|
||||
|
||||
public void setStartTime(Integer startTime) {
|
||||
this.startTime = startTime;
|
||||
}
|
||||
|
||||
public Integer getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
|
||||
public void setEndTime(Integer endTime) {
|
||||
this.endTime = endTime;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -494,5 +494,32 @@
|
||||
<artifactId>msgpack</artifactId>
|
||||
<version>0.6.12</version>
|
||||
</dependency>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>com.asprise.ocr</groupId>
|
||||
<artifactId>java-ocr-api</artifactId>
|
||||
<version>[15,)</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.java.dev.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
<version>4.1.0</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.sourceforge.tess4j</groupId>
|
||||
<artifactId>tess4j</artifactId>
|
||||
<version>3.4.0</version>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.sun.jna</groupId>
|
||||
<artifactId>jna</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
|
||||
+49
-35
@@ -68,20 +68,30 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("status", 1)
|
||||
// q.put("name","ÁÉÔ´Ò»´å")
|
||||
def dbcursor = LOU_DONG_STAT.find(q)
|
||||
def map = [:]
|
||||
def set = new HashSet()
|
||||
for(def db : dbcursor){
|
||||
|
||||
def list = map.get(db.region+"##"+db.name)
|
||||
if(!list){
|
||||
list = []
|
||||
map.put(db.region+"##"+db.name, list)
|
||||
}
|
||||
list << db
|
||||
if(db.house_num && db.house_num!="--"){
|
||||
def key = db.region+"##"+db.name +"##"+db.house_num
|
||||
if(!set.contains(key)){
|
||||
set << key
|
||||
list << db
|
||||
}
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap){
|
||||
def rets = []
|
||||
def keys = []
|
||||
distMap.each{k,dist->
|
||||
def buildings = buildingMap.get(k)
|
||||
if(buildings){
|
||||
@@ -89,53 +99,57 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(obj.region)
|
||||
if(areaData){
|
||||
ret << cityCodeDatas.get(obj.city)
|
||||
ret << obj._id.toString()
|
||||
ret << obj.house_num
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << areaData.district_code
|
||||
ret << dist._id.toString()
|
||||
if(obj.build_year){
|
||||
ret << obj.build_year
|
||||
}else{
|
||||
def key = dist._id.toString() +"##"+obj.house_num
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
ret << cityCodeDatas.get(obj.city)
|
||||
ret << obj._id.toString()
|
||||
ret << obj.house_num
|
||||
ret << obj.house_num
|
||||
ret << ""
|
||||
}
|
||||
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
if(arrs){
|
||||
ret << arrs[0].trim()
|
||||
}else{
|
||||
ret << ''
|
||||
}
|
||||
ret << areaData.district_code
|
||||
ret << dist._id.toString()
|
||||
if(obj.build_year){
|
||||
ret << obj.build_year
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
def arrs = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
if(arrs){
|
||||
ret << arrs[0].trim()
|
||||
}else{
|
||||
ret << ''
|
||||
}
|
||||
|
||||
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
|
||||
if(obj.floors){
|
||||
ret << obj.floors
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << CcbCodeUtils.getBuildStructureCode(dist.build_structure)
|
||||
if(obj.floors){
|
||||
ret << obj.floors
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
// if(obj.elevators && obj.elevators>=1){
|
||||
// ret << "1"
|
||||
// }else{
|
||||
// ret << "0"
|
||||
// }
|
||||
ret << ""
|
||||
ret << ""
|
||||
// if(obj.elevators && obj.elevators>=1) {
|
||||
// ret << obj.elevators//µçÌÝÊý
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << "1"
|
||||
rets << ret
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << "1"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+16
-5
@@ -27,7 +27,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
@@ -187,11 +187,22 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}else{
|
||||
def jsonObj = null
|
||||
if(dist.address){
|
||||
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
// jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
|
||||
jsonObj = OpenApiClient.standard1Action(dist.city, dist.address)
|
||||
}
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
if(jsonObj && jsonObj.street){
|
||||
if(jsonObj.street){
|
||||
ret << jsonObj.street
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(jsonObj.streetNumber){
|
||||
ret << jsonObj.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
|
||||
+1
@@ -508,6 +508,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
q.put("status", 0)
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
|
||||
+10
-3
@@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220418, 20220519);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
totalCount.addAndGet(1)
|
||||
def commId = rMap.get("_id").toString()
|
||||
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
|
||||
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
|
||||
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
|
||||
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
||||
@@ -408,7 +408,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
@@ -449,6 +449,13 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(estate_type2 && estate_type2=="ÆÕͨסլ"){
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("ÉÌס")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
+1
-1
@@ -367,7 +367,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","特房锦绣一方")
|
||||
// q.put("name","辽源一村")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["ĆŐͨץՏ","ÉĚץ"]))
|
||||
|
||||
|
||||
+6
@@ -57,6 +57,12 @@ class CcbFtpUtils {
|
||||
ftpUpload(dataCodeFileName)//上传加密码文件到FTP
|
||||
}
|
||||
|
||||
static def listFileNames(month){
|
||||
//上传到ftp
|
||||
sftpChannel = reConnect();
|
||||
Vector files = sftp.listFiles(ftpPath+ month+"/", sftpChannel);
|
||||
return files
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
|
||||
|
||||
+12
-11
@@ -15,8 +15,8 @@ class DealPriceDataProcess {
|
||||
|
||||
|
||||
|
||||
def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
// def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
@@ -25,8 +25,8 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
// task.deleteThirdPartyData()
|
||||
task.delDealByPe()
|
||||
task.deleteThirdPartyData()
|
||||
// task.delDealByPe()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
@@ -370,26 +370,27 @@ class DealPriceDataProcess {
|
||||
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = ["¶«Ý¸"] //,"º¼ÖÝ""¹ãÖÝ",
|
||||
def citys = CcbCommonUtils.citys
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def publishTime = '2021-10-01'
|
||||
def month = 202110
|
||||
def month = 202204
|
||||
def rows = []
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty(city, 202111)
|
||||
def rets = findPeGteThirdParty(city, 202205)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
for(def m : rets){
|
||||
def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
sqlInstance236.executeUpdate(sql)
|
||||
}
|
||||
println city +" "+total
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def findPeGteThirdParty(cityName, yearmonth){
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='µÚÈý·½' and cityname='${cityName}' and pe>20"
|
||||
def sql = "select commid,target_avg_price from avm_evaluation_item_gj where year_month=${yearmonth} and data_type='µÚÈý·½' and cityname='${cityName}' and pe>50"
|
||||
def rets = []
|
||||
sqlInstance2.eachRow(sql,{r->
|
||||
sqlInstance236.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
map.commid = r.commid
|
||||
map.price = r.target_avg_price
|
||||
|
||||
+2
-2
@@ -38,8 +38,8 @@ class ExportOracleData {
|
||||
exportCommunityBuileType()
|
||||
|
||||
exportThirdPartyPrice2(lastMonth)
|
||||
exportBigdataForsale("2022-02-19")
|
||||
exportBigdataDeal("2022-02-01")
|
||||
exportBigdataForsale("2022-04-19")
|
||||
exportBigdataDeal("2022-04-01")
|
||||
|
||||
|
||||
//计算结果相关
|
||||
|
||||
+23
@@ -3,8 +3,10 @@ package com.fangjia.script.ccbevaluation
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import groovy.sql.Sql
|
||||
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
|
||||
class ExportOracleEvaluationData {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
@@ -33,6 +35,8 @@ class ExportOracleEvaluationData {
|
||||
exportCommCaseDetailGj(curMonth)
|
||||
|
||||
def curDate = curMonth.toString()+"28"
|
||||
// 生成MD5文件
|
||||
exportMd5()
|
||||
SSHHelper jschCommand = new SSHHelper();
|
||||
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
|
||||
// okÎļþ
|
||||
@@ -413,4 +417,23 @@ class ExportOracleEvaluationData {
|
||||
CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate)
|
||||
}
|
||||
|
||||
def exportMd5(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
Vector files = CcbFtpUtils.listFileNames(curDate)
|
||||
Iterator fileVec = files.iterator()
|
||||
def fileRows = []
|
||||
while(fileVec.hasNext()){
|
||||
ChannelSftp.LsEntry file = fileVec.next()
|
||||
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
|
||||
fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
|
||||
}
|
||||
}
|
||||
def fileName = "/ccb/md5.txt"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\房价网\\md5.txt"
|
||||
}
|
||||
FileUtils.createTxtFile(fileRows, fileName, "utf-8", false);
|
||||
CcbFtpUtils.ftpUpload(null, fileName, curDate)
|
||||
println "md5.txt 上传Ftp完成......"
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2022-04-1", "2022-05-1")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
|
||||
+51
-1
@@ -1,21 +1,29 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class LoadGuide {
|
||||
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new LoadGuide()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
addPersonGuided("D:\\数据\\建行\\人工指导价更新.xlsx")
|
||||
// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
|
||||
// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx")
|
||||
@@ -26,9 +34,51 @@ class LoadGuide {
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx")
|
||||
savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
|
||||
// savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx")
|
||||
|
||||
}
|
||||
|
||||
def addPersonGuided(filename){
|
||||
ExcelUtils.modify(filename, { d ->
|
||||
def city = d['B']
|
||||
if(city){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
createSql( d, cityCodeDatas, distCodeMap)
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def createSql(d, cityCodeDatas, distCodeMap){
|
||||
def id = new ObjectId()
|
||||
def commid = d['A']
|
||||
def cityId = cityCodeDatas.get(d['B'])
|
||||
def sampleRegion = d['C'].replace("街道","").replace("镇","")
|
||||
def distRegion = distCodeMap.get(sampleRegion)
|
||||
def cityName = d['B']
|
||||
def distName = d['C']
|
||||
def commName = d['D']
|
||||
def price = d['E'].toDouble().toInteger()
|
||||
|
||||
def dist = CommonDatautils.findDistrict(cityName, sampleRegion, commName, "district")
|
||||
if(dist){
|
||||
def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
def dealDate = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd")
|
||||
|
||||
|
||||
def sql = "insert into guided_price_gj(ID,COMMID,COMMNAME,CITYID,CITYNAME,DISTID,DISTNAME,DEALDATE,AVG_PRICE,GUIDING_PERSON,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES " +
|
||||
"('${id.toString()}','${dist._id.toString()}', '${commName}','${cityId}','${cityName}','${distRegion.district_code}','${distName}',to_date('${dealDate}','yyyy-mm-dd'),${price},'房价网','',to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'),'FJW')"
|
||||
println sql
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}else{
|
||||
println " 没查到 "+commName
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
def match(def filename){
|
||||
def city = "西安"
|
||||
ExcelUtils.modify(filename,{d->
|
||||
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DistrictCodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* oracle中小区统计数据同步至85中
|
||||
*/
|
||||
class OracleDistrict2Mongo85 {
|
||||
|
||||
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection communityPoiConn = MongoUtils.getCollectionByName("fangjia_stat", "community_poi_gj", "192.168.0.85", 28017)
|
||||
|
||||
static main(args) {
|
||||
// 只写入重庆市数据
|
||||
def sql = "select cityid, distname, commid, commname, subway, traffic, bank, primary, junior, hospital, funeral from COMMUNITY_POI_GJ"
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def city = DistrictCodeUtils.getCityName(r['cityid'])
|
||||
def region = RegionData.formatRegion(city, r['distname'])
|
||||
DBObject dbObject = new BasicDBObject("_id", r['commid'])
|
||||
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
|
||||
.append("name", r['commname'])
|
||||
.append('subway', Integer.parseInt(r['subway'].toString()))
|
||||
.append("traffic", Integer.parseInt(r['traffic'].toString()))
|
||||
.append("bank", Integer.parseInt(r['bank'].toString()))
|
||||
.append("primary", Integer.parseInt(r['primary'].toString()))
|
||||
.append("junior", Integer.parseInt(r['junior'].toString()))
|
||||
.append("hospital", Integer.parseInt(r['hospital'].toString()))
|
||||
.append("funeral", Integer.parseInt(r['funeral'].toString()))
|
||||
.append("c_date", new Date());
|
||||
communityPoiConn.update(dbObject, updateObject, true, false)
|
||||
count++;
|
||||
if (count % 500 == 0) {
|
||||
println "已写入" + count + "条小区数据"
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DistrictCodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
/**
|
||||
* oracle中小区置信度数据同步至85中
|
||||
*/
|
||||
class OracleDistrictBasePrice2Mongo85 {
|
||||
|
||||
static def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static final DBCollection communityBasePriceConn = MongoUtils.getCollectionByName("business", "community_base_price_gj", "192.168.0.85", 28017)
|
||||
|
||||
static main(args) {
|
||||
def sql = "SELECT cityname, distname, commid, commname, confidence FROM community_base_price_gj WHERE YEAR_MONTH='202205'"
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
def city = r['cityname']
|
||||
def region = RegionData.formatRegion(city, r['distname'])
|
||||
DBObject dbObject = new BasicDBObject("_id", r['commid'])
|
||||
if (r['confidence'] != null) {
|
||||
def credit = new BigDecimal(r['confidence']).setScale(2, BigDecimal.ROUND_HALF_UP).toDouble()
|
||||
DBObject updateObject = new BasicDBObject("city", city).append("region", region)
|
||||
.append("name", r['commname'])
|
||||
.append("credit", credit)
|
||||
.append("c_date", new Date());
|
||||
communityBasePriceConn.update(dbObject, updateObject, true, false)
|
||||
count++;
|
||||
} else {
|
||||
print "置信度为空:" + r.toString()
|
||||
}
|
||||
if (count % 500 == 0) {
|
||||
println "已写入" + count + "条小区数据"
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
+71
-24
@@ -225,17 +225,42 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx"
|
||||
//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01")
|
||||
//def testMap = findCaluateCityPPE(sqlInstance236, 202202)
|
||||
//def dealMap = findDealCount(sqlInstance236, "2022-02-01", "2022-03-01")
|
||||
//def testMap = findCaluateCityPPE(sqlInstance236, 202203)
|
||||
//ExcelUtils.modify(fileName,{d->
|
||||
// if(d['C']){
|
||||
// def oldcount = dealMap.get(d['C'])
|
||||
// d['V'] = oldcount
|
||||
// d['X'] = oldcount
|
||||
// def casecount = testMap.get(d['C'])
|
||||
// d['W'] = casecount
|
||||
// d['Y'] = casecount
|
||||
// println d
|
||||
// }
|
||||
//},0)
|
||||
def rows = findTradePpeData(sqlInstance236)
|
||||
// rows.unique()
|
||||
def map=findTradeDetail(sqlInstance236,null)
|
||||
def datas=[]
|
||||
map.each{key,row->
|
||||
if(!rows.contains(key)){
|
||||
datas<<row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
|
||||
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def datas=[]
|
||||
//cityCodeDataMap.each{city,code->
|
||||
// def rows = findTradePpeData(sqlInstance236, 202203,code)
|
||||
// def alltrades=findTradeDetail(sqlInstance236,code)
|
||||
// alltrades.each{it->
|
||||
// def commid=it[0]
|
||||
// if(!rows.contains(commid)){
|
||||
// datas<<it
|
||||
// }
|
||||
// }
|
||||
// println datas.size()
|
||||
//}
|
||||
//ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交未使用.xlsx", datas, "数据")
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d->
|
||||
// if(d['I'] && d['K']){
|
||||
@@ -270,24 +295,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def
|
||||
// }
|
||||
//},0)
|
||||
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
|
||||
def flag = d['M']
|
||||
if(flag==0.0){
|
||||
if(d['D']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(d['D']))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
index++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
//def index = 0
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d->
|
||||
// def flag = d['M']
|
||||
// if(flag==0.0){
|
||||
// if(d['D']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_cat", 11)
|
||||
// udata.put("m_date",new Date())
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", new ObjectId(d['D']))
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// index++
|
||||
// println index
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d->
|
||||
// if(d['A']){
|
||||
@@ -558,8 +583,30 @@ def findCaluateCityPPE(def sqlInstance, def yearMonth){
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findTradePpeData(def sqlInstance){
|
||||
def sql = "SELECT commid,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate,target_avg_price FROM avm_evaluation_item_gj where year_month=202203 and data_type='成交'";
|
||||
def rows = new HashSet()
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
rows<<r.commid+"##"+r.buildarea+"##"+r.target_avg_price+"##"+r.dealdate
|
||||
});
|
||||
return rows
|
||||
}
|
||||
def findTradeDetail(def sqlInstance, def cityid){
|
||||
def sql = "select commid,cityname,distname,commname,buildarea,dealunitprice,to_char(dealdate,'yyyy-mm-dd') as dealdate from bigdata_deal_gj where dealdate>=to_date('2022-02-01','yyyy-MM-dd')";
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def key=r.commid+"##"+r.buildarea+"##"+r.dealunitprice+"##"+r.dealdate
|
||||
def row=[]
|
||||
row<<r.commid
|
||||
row<<r.cityname
|
||||
row<<r.distname
|
||||
row<<r.commname
|
||||
row<<r.buildarea
|
||||
row<<r.dealunitprice
|
||||
map.put(key,row)
|
||||
});
|
||||
return map
|
||||
}
|
||||
def findEstateType(cityid, sqlInstance){
|
||||
def sql = "select count(*) as count from bigdata_commbuildtype_gj where cityid='${cityid}' and buildtype!='0199'";
|
||||
def count = 0
|
||||
|
||||
+3
-3
@@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= 202202//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202112
|
||||
def endDate = 202202
|
||||
def sDate= 202204//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def startDate = 202202
|
||||
def endDate = 202204
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
+10
-10
@@ -40,8 +40,8 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","2月总量","3月总量","4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-12-01"
|
||||
def date = "2022-01-01"
|
||||
def endDate = "2022-02-01"
|
||||
def newDate = "2022-03-01"
|
||||
def startDate = "2022-02-01"
|
||||
def date = "2022-03-01"
|
||||
def endDate = "2022-04-01"
|
||||
def newDate = "2022-05-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
@@ -131,7 +131,7 @@ class TradeStatLog {
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据")
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-04成交统计.xlsx", rets, "数据")
|
||||
println "表格生成完成"
|
||||
}
|
||||
|
||||
@@ -481,8 +481,8 @@ class TradeStatLog {
|
||||
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
|
||||
def rets = []
|
||||
rets << headers
|
||||
def startDate = "2022-02-01"
|
||||
def endDate = "2022-03-01"
|
||||
def startDate = "2022-03-01"
|
||||
def endDate = "2022-04-01"
|
||||
for(def city : citys){
|
||||
|
||||
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
|
||||
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
class CcbDataStat {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDataStat()
|
||||
// task.run()
|
||||
task.tradeRun()
|
||||
}
|
||||
|
||||
def tradeRun(){
|
||||
def ccbCitys = CcbCommonUtils.citys;
|
||||
def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd")
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\成交案例统计_0531.xlsx", { d ->
|
||||
if(d['A']){
|
||||
if(ccbCitys.contains(d['A'])){
|
||||
d['D'] = findOracleTotalCount(d['A'])
|
||||
}else{
|
||||
def totalcount = findAllTradeCount(d['A'], startDate, endDate)
|
||||
def repeatCount = findRepeatCount(d['A'], startDate, endDate)
|
||||
def matchTotalCount = findMatchTradeCount(d['A'], startDate, endDate)
|
||||
def matchRepeatTotalCount = findMatchRepeatTradeCount(d['A'], startDate, endDate)
|
||||
d['B'] = totalcount
|
||||
d['C'] = repeatCount
|
||||
d['D'] = matchTotalCount
|
||||
d['E'] = matchRepeatTotalCount
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findRepeatCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def set = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def key = d.region+"##"+d.district_name+"##"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
|
||||
if(!set.contains(key)){
|
||||
set << key
|
||||
}
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findMatchRepeatTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', true))
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def set = new HashSet()
|
||||
for(def d : dbcursor){
|
||||
def key = d.match_id+"#"+d.avg_price+"##"+d.area+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyyMMdd")
|
||||
if(!set.contains(key)){
|
||||
set << key
|
||||
}
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
|
||||
def findMatchTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', true))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
public static def findAllTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
// q.put("match_id", new BasicDBObject('$exists', false))
|
||||
def total = DEAL_PRICE_COLL.count(q)
|
||||
return total
|
||||
}
|
||||
|
||||
def findOracleTotalCount(city){
|
||||
def sql = "select count(*) as count from bigdata_deal_gj where cityname='${city}' and dealdate>=to_date('2021-01-01','yyyy-MM-dd') and dealdate<to_date('2022-01-01','yyyy-MM-dd')";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def run(){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
|
||||
if(d['A']) {
|
||||
def citycode = cityCodeDataMap.get(d['A'])
|
||||
def total = findTotalCount(citycode)
|
||||
def f_count = findDistFinishDate(citycode)
|
||||
def b_count = findBuildTypeCount(citycode)
|
||||
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
|
||||
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
|
||||
d['C'] = f_pre
|
||||
d['D'] = b_pre
|
||||
d['E'] = total
|
||||
println d['A'] +" "+ total +" "+f_count+" "+b_count
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def findTotalCount(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findDistFinishDate(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and BUILDYEAR>0";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findBuildTypeCount(citycode){
|
||||
def sql = "select count(*) as count from (select commid, count(commid) as count from bigdata_commbuildtype_gj where cityid='${citycode}' and BUILDTYPE!='0199' group by commid)";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import groovy.sql.Sql
|
||||
|
||||
class CcbDistNameMatch {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDistNameMatch();
|
||||
// task.run()
|
||||
task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
if(d['D'].contains("、")){
|
||||
println d['D']
|
||||
}
|
||||
def arrs = d['D'].split("、")
|
||||
def list = []
|
||||
def flags = []
|
||||
for(def s : arrs){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, s, "district")
|
||||
if(dist){
|
||||
list << dist.name
|
||||
flags << "完全一致"
|
||||
}else{
|
||||
def jsonObject = OpenApiClient.addressMatch(d['A'], s, "property", false, newRegion)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
if(jsonObject.get("district") == newRegion){
|
||||
list << jsonObject.get("name")
|
||||
flags << "匹配接口"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if(list){
|
||||
list = list.unique()
|
||||
d['E'] = list.join(",")
|
||||
d['F'] = flags.join(",")
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\不同小区.xlsx", { d ->
|
||||
def regionCode = null
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if(dist){
|
||||
d['D'] = dist.name
|
||||
def count = findDistrictByOracle(dist._id.toString())
|
||||
if(count && count>0){
|
||||
d['F'] = 1
|
||||
}else{
|
||||
d['H'] = dist.estate_type2
|
||||
d['J'] = dist.name_cat
|
||||
}
|
||||
}else{
|
||||
def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district")
|
||||
if (rets) {
|
||||
if(rets.size()>1){
|
||||
d['E'] = "别名-多个"
|
||||
}else{
|
||||
dist = rets[0]
|
||||
d['E'] = d['C'] +"--"+dist.name
|
||||
def count = findAliasByOracle(dist._id.toString())
|
||||
if(count && count>0) {
|
||||
d['G'] = 1
|
||||
}else{
|
||||
d['H'] = dist.estate_type2
|
||||
d['J'] = dist.name_cat
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def findDistrictByOracle(commid){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where commid='${commid}' and status=0";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
def findAliasByOracle(commid){
|
||||
def sql = "select count(*) as count from address_alias_fjw where address_id='${commid}'";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
}
|
||||
+305
-5
@@ -2,25 +2,248 @@ package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class CcbDistrictToDB {
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDistrictToDB();
|
||||
// task.run()
|
||||
task.run1()
|
||||
// task.run1()
|
||||
// task.meituanRun()
|
||||
task.meituanSqrentRun()
|
||||
// task.updateDistrict()
|
||||
// task.updateDongguangRegion()
|
||||
}
|
||||
|
||||
def updateDongguangRegion(){
|
||||
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
|
||||
def dist = CommonDatautils.findAllDistrictByCity("东莞", "district", ["city","region","name","c_date","estate_type2"])
|
||||
for(def d : dist){
|
||||
def t = d.c_date
|
||||
if(t.getTime()>cDate.getTime()){
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateDistrict(){
|
||||
def citys = ["惠州","东莞","金华","苏州","桂林","潮州","廊坊","临沧","通辽","雅安"]
|
||||
def cDate = DateUtils.parseFromTightByPattern("2022-04-27","yyyy-MM-dd")
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
def dist = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","c_date","estate_type2"])
|
||||
for(def d : dist){
|
||||
def t = d.c_date
|
||||
if(t.getTime()>cDate.getTime()){
|
||||
if(d.name.contains("(")){
|
||||
def delstr = d.name.substring(d.name.indexOf("("), d.name.indexOf(")")+1)
|
||||
def newName = d.name.replace(delstr,"")
|
||||
index++
|
||||
println index+" "+d.city+" "+d.name+" "+newName
|
||||
def obj = CommonDatautils.findDistrict(d.city, d.region, newName, "district")
|
||||
if(!obj){
|
||||
println "update"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name", newName)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
|
||||
d.name = newName
|
||||
addPrice(d)
|
||||
|
||||
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
CommonUtils.recalculationSqRent(d.city, d.region, newName, d.estate_type2, lastMonthDay,false,0)
|
||||
}else{
|
||||
println "delete"
|
||||
DbUtils.seaweed.remove(new BasicDBObject("_id",d._id))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def meituanSqrentRun(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
|
||||
index++
|
||||
println index+" "+d['H']
|
||||
def city = d['F']
|
||||
def region = d['G']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion) {
|
||||
def name = NameProcess.nameStandard(city, newRegion, d['H'])
|
||||
name = nameFileter(name)
|
||||
|
||||
if(name){
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
CommonUtils.recalculationSqRent(city, region, name, estateType2, lastMonthDay,false,0)
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def meituanRun(){
|
||||
def index = 0//美团需入库小区合并后.xlsx
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\美团\\美团待入库.xlsx", { d ->
|
||||
index++
|
||||
println index+" "+d['H']
|
||||
def city = d['F']
|
||||
def region = d['G']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion) {
|
||||
def name = NameProcess.nameStandard(city, newRegion, d['H'])
|
||||
name = nameFileter(name)
|
||||
|
||||
if (name) {
|
||||
def formatAddress = null
|
||||
if (d['I']) {
|
||||
formatAddress = AddressUtil.formatAddress(city, d['I'])
|
||||
}
|
||||
|
||||
// if(!formatAddress || (formatAddress && !formatAddress.contains("号") && !formatAddress.contains("交叉"))){
|
||||
// def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name)
|
||||
// def map = findStandardObject2(city,region, name, d['E'], rets);
|
||||
// if(map && map.address){
|
||||
// formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
// if(!formatAddress){
|
||||
// if(map.address.contains("交叉口")){
|
||||
// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉口")+3)
|
||||
// }else if(map.address.contains("交叉")){
|
||||
// formatAddress = map.address.substring(0, map.address.toString().indexOf("交叉")+2)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if (d['L']) {
|
||||
glat = d['M'].toString().toDouble() * 10000000000L
|
||||
glng = d['L'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if (name.contains("别墅")) {
|
||||
estateType2 = "别墅"
|
||||
} else if (name.contains("商住")) {
|
||||
estateType2 = "商住"
|
||||
} else {
|
||||
def type = d['J']
|
||||
if (type) {
|
||||
if (type.contains("别墅")) {
|
||||
estateType2 = "别墅"
|
||||
} else if (type.contains("商住")) {
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city + "##" + newRegion + "##" + name
|
||||
if (name && !keys.contains(key)) {
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if (!dist) {
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat", 12)
|
||||
data.put("status", 0)
|
||||
data.put("cat", "district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region", newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2", estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if (glat) {
|
||||
data.put("lat", glat)
|
||||
data.put("lng", glng)
|
||||
data.put("lat2", blat)
|
||||
data.put("lng2", blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map", markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
|
||||
|
||||
println index + " " + data
|
||||
|
||||
def stop = false
|
||||
while (!stop) {
|
||||
if (data._id.toString()) {
|
||||
addPrice(data)
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-05-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-24")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def run1(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增小区数据_1236.xlsx", { d ->
|
||||
def city = d['A']
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
@@ -71,6 +294,7 @@ class CcbDistrictToDB {
|
||||
if(name){
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district")
|
||||
if(!dist){
|
||||
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
@@ -216,9 +440,85 @@ class CcbDistrictToDB {
|
||||
}
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
if(name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
}
|
||||
|
||||
return name;
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+78
-1
@@ -1,10 +1,12 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
@@ -13,7 +15,82 @@ class DistrictMarge {
|
||||
static main(args) {
|
||||
def task = new DistrictMarge()
|
||||
//task.run()
|
||||
task.repeatAddress()
|
||||
// task.repeatAddress()
|
||||
// task.removeDist()
|
||||
task.updateAddress()
|
||||
// task.addMappingAlias()
|
||||
}
|
||||
def updateAddress(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\更新小区地址.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
def region = d['B'].trim()
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("address", d['D'])
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
}else{
|
||||
println d['A'] + " 格式化失败 "+d['B']
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def removeDist(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\删除小区.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
}else{
|
||||
println d['A'] + " 格式化失败 "+d['B']
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addMappingAlias(){
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\新增映射别名.xlsx", { d ->
|
||||
index ++
|
||||
if (d['A']) {
|
||||
def region = d['B'].trim()
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if (dist) {
|
||||
def mapping_alias = dist.mapping_alias
|
||||
if(!dist.mapping_alias){
|
||||
mapping_alias = []
|
||||
}
|
||||
def alias = d['D']
|
||||
def arrs = alias.split("、")
|
||||
for(def s : arrs){
|
||||
if(!mapping_alias.contains(s)){
|
||||
mapping_alias << s
|
||||
}
|
||||
}
|
||||
DBObject udata = new BasicDBObject()
|
||||
udata.put("mapping_alias", mapping_alias)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
}else{
|
||||
println d['A'] + " 格式化失败 "+d['B']
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def repeatAddress(){
|
||||
|
||||
+28
-1
@@ -36,10 +36,37 @@ class MatchMapData {
|
||||
// task.run3()
|
||||
// task.exportHouse()
|
||||
// task.addDistrictLevel()
|
||||
task.run4()
|
||||
// task.run4()
|
||||
task.run5()
|
||||
|
||||
}
|
||||
|
||||
def run5(){
|
||||
ExcelUtils.multiThreadingModify("D:\\Êý¾Ý\\½¨ÐÐ\\Êý¾ÝºËʵ.xlsx", { d ->
|
||||
if (d['A']) {
|
||||
def region = d['B']
|
||||
def newRegion = RegionData.formatRegion(d['A'], region)
|
||||
if (newRegion) {
|
||||
def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district")
|
||||
if(dist){
|
||||
if(dist.mapping_alias){
|
||||
d['D'] = dist.mapping_alias.join(",")
|
||||
}
|
||||
d['E'] = dist.address
|
||||
}
|
||||
|
||||
def dist1 = CommonDatautils.findDistrict(d['A'], newRegion, d['G'], "district")
|
||||
if(dist1){
|
||||
if(dist1.mapping_alias){
|
||||
d['H'] = dist1.mapping_alias.join(",")
|
||||
}
|
||||
d['I'] = dist1.address
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run4() {
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d ->
|
||||
def regionCode = null
|
||||
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
package com.fangjia.script.ccbevaluation.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class OracleHouse {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new OracleHouse()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def rets = findAllHouse("2022-04-19")
|
||||
def totalCount = new AtomicLong()
|
||||
println rets.size()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
for(def i=0;i<rets.size();i++){
|
||||
def arrs = rets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def flag = findHouseFlag(arrs[0])
|
||||
println totalCount.get()+" "+ arrs[0] +" "+flag
|
||||
if(flag){
|
||||
def dist = CommonDatautils.findDistrictById(arrs[1])
|
||||
if(dist && dist.estate_type2=="ÆÕͨסլ"){
|
||||
removeHouse(arrs[0])
|
||||
}
|
||||
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouseFlag(def id){
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_comp_potential", queryBuilder);
|
||||
def delFlag = false
|
||||
for(def h : houses){
|
||||
if(h.estate_type && (h.estate_type.contains("ÉÌ") || h.estate_type.contains("°ì"))){
|
||||
println id+" "+h.estate_type
|
||||
delFlag = true
|
||||
}
|
||||
}
|
||||
return delFlag
|
||||
}
|
||||
|
||||
def findAllHouse(def publishDate) {
|
||||
def sql = "select FORSALEID,COMMID from bigdata_forsale_gj where publishtime>=to_date('${publishDate}','yyyy-MM-dd')";
|
||||
def allRows = []
|
||||
def index = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
index = index + 1
|
||||
println index
|
||||
def data = []
|
||||
data << r.FORSALEID
|
||||
data << r.COMMID
|
||||
allRows << data
|
||||
});
|
||||
return allRows
|
||||
}
|
||||
|
||||
def removeHouse(houseid){
|
||||
def sql = "delete from bigdata_forsale_gj where FORSALEID='${houseid}' ";
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
+277
@@ -0,0 +1,277 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
/**
|
||||
* 重庆估值测试--2小时
|
||||
*/
|
||||
class EvaluationBatch {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new EvaluationBatch()
|
||||
// task.run()
|
||||
// task.run1();
|
||||
task.run2()
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2021-01-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-07")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
ExcelUtils.modify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V1.xlsx", { row ->
|
||||
if(row['N'] && row['N'].toString().toDouble()>=20){
|
||||
println row['F'] +" "+row['N']
|
||||
def dist = CommonDatautils.findDistrictById(row['C'])
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(row['D'], row['E'], row['F'], dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
// FangjiaClientUtils.sendDistrictPriceCalculate(row['D'], row['E'], row['F'], 48);
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def city = "重庆"
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆房屋交易信息V2.xlsx", { row ->
|
||||
def region = row['E']
|
||||
def name = row['F']
|
||||
def id = row['C']
|
||||
def lastWeekend = DateUtils.getFirstDayOfWeek(new Date()-30)
|
||||
def lastYearWeekend = DateUtils.getFirstDayOfWeek(new Date()-365)
|
||||
if (region && name) {
|
||||
row['H'] = CommonDatautils.findNewSqmetreRent(city, region, name)
|
||||
|
||||
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
|
||||
if(priceObj){
|
||||
row['AB'] = priceObj.avg_price
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.findDistrictById(id)
|
||||
if(dist) {
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(dist)
|
||||
def map = getArroundAvgPriceMap(list)
|
||||
if(map){
|
||||
row['L'] = map.avg_price
|
||||
row['Q'] = map.max_val
|
||||
row['R'] = map.min_val
|
||||
}
|
||||
def lastArroundPrice = getArroundAvgPriceByWeekend(list, lastWeekend)
|
||||
if(lastArroundPrice && priceObj){
|
||||
def mod = (map.avg_price-lastArroundPrice)*100/lastArroundPrice
|
||||
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||
row['P'] = mod
|
||||
}
|
||||
def lastYearArroundPrice = getArroundAvgPriceByWeekend(list, lastYearWeekend)
|
||||
if(lastYearArroundPrice && priceObj){
|
||||
def mod = (map.avg_price-lastYearArroundPrice)*100/lastYearArroundPrice
|
||||
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||
row['O'] = mod
|
||||
}
|
||||
}
|
||||
def lastWeekendObj = CommonDatautils.queryDistrictPrice(city, region, name, lastWeekend)
|
||||
if(lastWeekendObj){
|
||||
row['S'] = lastWeekendObj.avg_price
|
||||
if(priceObj){
|
||||
def mod = (priceObj.avg_price-lastWeekendObj.avg_price)*100/lastWeekendObj.avg_price
|
||||
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||
row['M'] = mod
|
||||
}
|
||||
}
|
||||
def lastYearObj = CommonDatautils.queryDistrictPrice(city, region, name, lastYearWeekend)
|
||||
if(lastYearObj){
|
||||
row['T'] = lastYearObj.avg_price
|
||||
if(priceObj){
|
||||
def mod = (priceObj.avg_price-lastYearObj.avg_price)*100/lastYearObj.avg_price
|
||||
mod = DataHandleUtils.getDoubleRound(2, mod)
|
||||
row['N'] = mod
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
def city = "重庆"
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆估值交易信息V2.xlsx", { row ->
|
||||
def region = row['E']
|
||||
def name = row['F']
|
||||
def id = row['C']
|
||||
def area = row['G'].toDouble()
|
||||
if(region && name){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
|
||||
if(priceObj){
|
||||
row['J'] = priceObj.avg_price
|
||||
if(row['C']){
|
||||
row['I'] = (area*priceObj.avg_price).toInteger()
|
||||
}
|
||||
}
|
||||
def dist = CommonDatautils.findDistrictById(id)
|
||||
if(dist){
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(dist)
|
||||
row['N'] = getArroundAvgPrice(list)
|
||||
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
alias << dist.name
|
||||
def houseAvgPrice = findSimAreaHousePrice(dist.city, dist.region, alias,20220101,20220401, area-10, area+10)
|
||||
if(houseAvgPrice){
|
||||
row['K'] = houseAvgPrice
|
||||
}
|
||||
|
||||
def tradeAvgPrice = findTrades(dist, DateUtils.parseFromTightYYYYMMDD("2021-12-01"), area-10, area+10)
|
||||
row['L'] = tradeAvgPrice
|
||||
|
||||
def avg_price = findAvgPrice(dist._id.toString())
|
||||
if(row['J'] && avg_price){
|
||||
def weekendPrice = row['J'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
row['P'] = ppe15
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def findTrades(dist, startDate, startArea, endArea){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
// q.put("source", "链家在线")
|
||||
q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"]))
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||
q.put("area", new BasicDBObject('$gte', startArea).append('$lt',endArea))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def prices = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
prices << d.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
def price = null
|
||||
if(prices){
|
||||
price = (prices.sum()/prices.size()).toInteger()
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
|
||||
def getArroundAvgPriceMap(def list){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
prices.sort()
|
||||
def map = [:]
|
||||
map.min_val = prices[0]
|
||||
map.max_val = prices[prices.size()-1]
|
||||
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
map.avg_price = avg_price
|
||||
return map;
|
||||
}
|
||||
|
||||
def getArroundAvgPrice(def list){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
|
||||
def getArroundAvgPriceByWeekend(def list, def weekend){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj =CommonDatautils.queryDistrictPrice(l.city, l.region, l.name, weekend)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
def avg_price = null;
|
||||
if(prices){
|
||||
avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
}
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
|
||||
def findSimAreaHousePrice(def city, def region, def alias, def startDate, def endDate, def startArea, def endArea){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
def prices = []
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and area>=${startArea} and area<=${endArea}"
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
prices << data.avg_price
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
def price = null
|
||||
if(prices){
|
||||
price = (prices.sum()/prices.size()).toInteger()
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
public String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
class BlankMatch {
|
||||
static main(args) {
|
||||
def task = new BlankMatch()
|
||||
task.run()
|
||||
// task.run1()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
}
|
||||
|
||||
def run() {
|
||||
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||
//def path = "D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx";
|
||||
def path = "D:\\数据\\重庆农商行\\2022-04-02\\测试.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['C']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['C'] = jsonObject.get("id")
|
||||
row['D'] = jsonObject.get("city")
|
||||
row['E'] = jsonObject.get("district")
|
||||
row['F'] = jsonObject.get("name")
|
||||
row['G'] = jsonObject.get("address")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run1() {
|
||||
//rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null);
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx", { row ->
|
||||
if(row['F']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['F'], "property", false, row['E'], null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['C'] = jsonObject.get("id")
|
||||
row['D'] = jsonObject.get("city")
|
||||
row['J'] = jsonObject.get("name")
|
||||
}
|
||||
|
||||
}
|
||||
// def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], "property", false, null, null)
|
||||
// if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// row['C'] = jsonObject.get("id")
|
||||
// row['D'] = jsonObject.get("city")
|
||||
// row['E'] = jsonObject.get("district")
|
||||
// row['F'] = jsonObject.get("name")
|
||||
// row['G'] = jsonObject.get("address")
|
||||
// }
|
||||
});
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
def dist = CommonDatautils.findDistrict("重庆", row['E'], row['F'], "district")
|
||||
if(dist){
|
||||
row['C'] = dist._id.toString()
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
def run2(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配不上(合并).xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\2022-04-02\\重庆样本匹配结果V3.xlsx", { row ->
|
||||
if(!row['C']){
|
||||
for(def d : datas){
|
||||
if(d['B'] == row['B']){
|
||||
row['C'] = d['C']
|
||||
row['D'] = d['D']
|
||||
row['E'] = d['E']
|
||||
row['F'] = d['J']
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class FieldAddValue {
|
||||
static main(args) {
|
||||
def task = new FieldAddValue()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
//16991 14360
|
||||
def districts = CommonDatautils.findAllDistrictByCity("ÖØÇì", "district", ["region","name","alias","is_elevator"])
|
||||
def index = 0
|
||||
def updateIndex = 0
|
||||
for(def d : districts){
|
||||
if(!d.is_elevator){
|
||||
index++
|
||||
println index
|
||||
def alias = d.alias
|
||||
if(!d.alias){
|
||||
alias = []
|
||||
}
|
||||
alias << d.name
|
||||
def maxHeight = findHouseHeight("ÖØÇì", d.region, alias, 20220101)
|
||||
if(maxHeight && maxHeight>6){
|
||||
updateIndex++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("is_elevator", true)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId( d._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
println index+" "+updateIndex
|
||||
}
|
||||
|
||||
def findHouseHeight(def city, def region, def alias, def startDate){
|
||||
def aliasStr = null;
|
||||
def total = 0
|
||||
def heights = null
|
||||
try{
|
||||
aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select max(height) as height from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and region='${region}' and district_name in ('${aliasStr}') "
|
||||
|
||||
ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
if(data.height){
|
||||
heights = data.height
|
||||
}
|
||||
|
||||
})
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
// def max_height = null
|
||||
// if(heights){
|
||||
// heights.sort()
|
||||
// max_height = heights[heights.size()-1]
|
||||
// }
|
||||
return heights
|
||||
}
|
||||
|
||||
public String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+612
@@ -0,0 +1,612 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.google.gson.JsonObject
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class MatchEvaluateRegionTest {
|
||||
static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/"
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def path = "D:\\数据\\美团\\10城小区明细-V10.xlsx"
|
||||
def total = 1500
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateRegionTest()
|
||||
// task.run()//小区匹配
|
||||
// task.run1()//地图匹配
|
||||
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
// task.addPrice()
|
||||
// task.addArroundPrice()
|
||||
// task.addArroundPriceByLat()
|
||||
// task.addRentRate()
|
||||
// task.addAuthor()
|
||||
// task.addLevel()
|
||||
// task.addSorce()
|
||||
// task.addArroundScore()
|
||||
|
||||
task.pkPrice()
|
||||
|
||||
}
|
||||
def run(){
|
||||
|
||||
def index = 0
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
index ++
|
||||
if(!row['F']){
|
||||
println index+" "+row['E']
|
||||
def jsonObject = OpenApiClient.addressMatch(row['A'], row['C'], "property", false,row['B'],null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = jsonObject.get("id")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("name")
|
||||
row['J'] = jsonObject.get("address")
|
||||
row['K'] = jsonObject.get("propertyType")
|
||||
row['L'] = jsonObject.get("map")
|
||||
row['M'] = jsonObject.get("lng")
|
||||
row['N'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F'] && row['A']){
|
||||
def jsonObject = OpenApiClient.addressMatch(row['A'], row['E'], null, true, row['B'],row['C'].toString())
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['F'] = jsonObject.get("id")
|
||||
row['G'] = jsonObject.get("city")
|
||||
row['H'] = jsonObject.get("district")
|
||||
row['I'] = jsonObject.get("name")
|
||||
row['J'] = jsonObject.get("address")
|
||||
row['K'] = jsonObject.get("propertyType")
|
||||
row['L'] = jsonObject.get("map")
|
||||
row['M'] = jsonObject.get("lng")
|
||||
row['N'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def pkPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['R'] && row['R'].toString().toDouble().toInteger()==1){
|
||||
def obj = CommonDatautils.queryDistrictPrice(row['G'], row['H'], row['I'], DateUtils.parseFromTightByPattern("2022-05-22","yyyy-MM-dd"))
|
||||
if(!obj){
|
||||
row['R'] = ''
|
||||
row['S'] = "无最新价"
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['L']){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(row['G'], row['H'], row['I'])
|
||||
if(priceObj){
|
||||
row['O'] = priceObj.avg_price
|
||||
// row['N']= (int)(priceObj.avg_price * row['E'])
|
||||
}
|
||||
|
||||
def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['G'], row['H'], row['I'])
|
||||
if(sqmetreRentPrice){
|
||||
row['P'] = sqmetreRentPrice
|
||||
}
|
||||
|
||||
// def avg_price = findAvgPrice(row['F'])
|
||||
// if(row['M'] && avg_price){
|
||||
// def weekendPrice = row['M'].toString().toDouble()
|
||||
// def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
// row['O'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
|
||||
// }
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def addRentRate(){
|
||||
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
|
||||
def radioMap = [:]
|
||||
for(def city : citys) {
|
||||
def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
|
||||
radioMap.put(city, avg_radio)
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def city = row['G']
|
||||
if (row['F'] && row['O'] && !row['P']) {
|
||||
def avg_rent_radio = radioMap.get(city)
|
||||
if(avg_rent_radio){
|
||||
row['P'] = (int)(row['O']/avg_rent_radio)
|
||||
row['S'] = "租金推算1"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addArroundPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['O']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['E'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['O'] = priceObj.unitPrice
|
||||
// row['N'] = (int) (priceObj.unitPrice * row['E'])
|
||||
row['S'] = "周边价格"
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addAuthor(){
|
||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['B']){
|
||||
if(index < avg){
|
||||
row['N'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { d->
|
||||
def flag = d['M']
|
||||
if(!flag){
|
||||
def city = d['E']
|
||||
def region = d['F']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['G']
|
||||
name = nameFileter(name)
|
||||
|
||||
def formatAddress = null
|
||||
def address = d['H']
|
||||
if(address){
|
||||
formatAddress = AddressUtil.formatAddress(city, address)
|
||||
if(!formatAddress){
|
||||
if(address.contains("交叉口")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
|
||||
}else if(address.contains("交叉")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city+"##"+newRegion+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
addPrice(data)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
if(name.contains("(")){
|
||||
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
|
||||
name = name.replace(delstr,"")
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
|
||||
def addSorce(){
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(d['F']){
|
||||
def score = findDistrictScore(d['F'])
|
||||
if(score){
|
||||
d['R'] = score
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addArroundScore(){
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['R'] && d['F']){
|
||||
def dist = null
|
||||
if(d['F'].toString().size()>16){
|
||||
dist = CommonDatautils.findDistrictById(d['F'])
|
||||
}
|
||||
|
||||
if(!dist){
|
||||
dist = new BasicDBObject()
|
||||
dist.city = d['G']
|
||||
dist.lng = d['M'].toString().toDouble()*10000000000L
|
||||
dist.lat = d['N'].toString().toDouble()*10000000000L
|
||||
}
|
||||
if(dist.lat){
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(dist)
|
||||
if (list) {
|
||||
list = sortByDistance(dist, list);
|
||||
if(list && list.size()>10){
|
||||
list = list[0..9]
|
||||
}
|
||||
if(list){
|
||||
def scores = []
|
||||
for(def s : list){
|
||||
def score = findDistrictScore(s._id.toString())
|
||||
if(score){
|
||||
scores << score
|
||||
}
|
||||
}
|
||||
def arround_score =(int)(scores.sum()/scores.size())
|
||||
d['R'] = arround_score
|
||||
d['T'] = "周边小区评分2"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addArroundPriceByLat(){
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(!d['O'] && d['F'] && (d['K'].contains("住宅") || d['K'].contains("商住") ||d['K'].contains("别墅"))){
|
||||
def dist = null
|
||||
if(d['F'].toString().size()>16){
|
||||
dist = CommonDatautils.findDistrictById(d['F'])
|
||||
}
|
||||
|
||||
if(!dist){
|
||||
dist = new BasicDBObject()
|
||||
dist.city = d['G']
|
||||
dist.lng = d['M'].toString().toDouble()*10000000000L
|
||||
dist.lat = d['N'].toString().toDouble()*10000000000L
|
||||
}
|
||||
if(dist.lat){
|
||||
List<DBObject> list = DbUtils.findAroundDistrict(dist)
|
||||
if (list) {
|
||||
list = sortByDistance(dist, list);
|
||||
if(list && list.size()>10){
|
||||
list = list[0..9]
|
||||
}
|
||||
if(list){
|
||||
def prices = []
|
||||
for(def s : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
if(prices){
|
||||
def arround_price =(int)(prices.sum()/prices.size())
|
||||
d['O'] = arround_price
|
||||
d['S'] = "周边均价3"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def sortByDistance(d, list){
|
||||
def newList = []
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
newList << nb
|
||||
}
|
||||
//排序
|
||||
Collections.sort(newList, new Comparator() {
|
||||
@Override
|
||||
int compare(Object o1, Object o2) {
|
||||
if(o1.distance > o2.distance)
|
||||
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
|
||||
if(o1.distance == o2.distance)
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
|
||||
return newList;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findDistrictScore(id){
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("_id", id))
|
||||
def indexObj = ElasticUtils.executeSimpleQuery("192.168.0.12", "district_comp_potential", queryBuilder);
|
||||
def score = 0
|
||||
for(def h : indexObj){
|
||||
score = h.total_price_rate
|
||||
if(score){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
|
||||
def addLevel() {
|
||||
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def cityLevelMap = [:]
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def datas = addPriceData(dists, priceMap);
|
||||
datas = calcuteLevel(datas);
|
||||
cityLevelMap.put(city, ["price":priceMap, "level":datas])
|
||||
println city +" 楼盘评级同步完成"
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
|
||||
def address = row['E']
|
||||
def city = RegionData.formatCity(row['A'])
|
||||
|
||||
if(row['K'] && !row['K'].contains("住宅") && !row['K'].contains("商住") && !row['K'].contains("别墅")){
|
||||
if(city){
|
||||
def map = cityLevelMap.get(city)
|
||||
def rMap = null
|
||||
if(map){
|
||||
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
if(rMap.rank){
|
||||
row['Q'] = rMap.rank
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPriceData(dists, priceMap){
|
||||
def newLists = []
|
||||
for(def d : dists){
|
||||
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
|
||||
if(price){
|
||||
d.avg_price = price;
|
||||
newLists << d;
|
||||
}
|
||||
}
|
||||
if(newLists){
|
||||
newLists.sort {a,b->
|
||||
def v1 = a.avg_price
|
||||
def v2 = b.avg_price
|
||||
return v1.compareTo(v2)
|
||||
}
|
||||
}
|
||||
return newLists;
|
||||
}
|
||||
|
||||
def calcuteLevel(def datas){
|
||||
def total = datas.size()
|
||||
def v = (int)(total/10)
|
||||
def start = 0;
|
||||
def end = 0;
|
||||
def map = [:]
|
||||
for(int i=1;i<=10;i++){
|
||||
end = end + v;
|
||||
if(i == 10){
|
||||
map.put(i, [start, total])
|
||||
}else{
|
||||
map.put(i, [start, end])
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
for(def d=0;d<datas.size();d++){
|
||||
def dist = datas[d]
|
||||
|
||||
for (def o in map) {
|
||||
def ranges = o.value
|
||||
def m = ranges[0]
|
||||
def n = ranges[1]
|
||||
if(d==0){
|
||||
if(d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}else{
|
||||
if(d>m && d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getArroundDistrictPriceByAddress(def city, def address, def levels){
|
||||
def json = GaoDeApiUtils.geo(city, address)
|
||||
def location = null
|
||||
def detail = new BasicDBObject()
|
||||
def level = null
|
||||
def rank = null
|
||||
if(json && json.count>=1){
|
||||
def m = json.geocodes[0]
|
||||
if(m){
|
||||
location = m.location
|
||||
level = m.level
|
||||
}
|
||||
}
|
||||
if(location){
|
||||
def arrs = location.split(",")
|
||||
detail.lng = arrs[0].toString().toDouble()*10000000000L
|
||||
detail.lat = arrs[1].toString().toDouble()*10000000000L
|
||||
}
|
||||
if(detail.lng){
|
||||
detail.city = city
|
||||
|
||||
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
|
||||
if(list && list.size()>10){
|
||||
list = sortByDistance(detail, list);
|
||||
try{
|
||||
list = list[0..9]
|
||||
}catch(Exception e){
|
||||
println list.size()
|
||||
}
|
||||
|
||||
}
|
||||
def arroundAvgPrice = getArroundAvgPrice(list)
|
||||
if(arroundAvgPrice){
|
||||
|
||||
def mod = Integer.MAX_VALUE;
|
||||
for(def d : levels){
|
||||
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
|
||||
if(newMod<mod){
|
||||
mod = newMod
|
||||
rank = d.level
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def m = [:]
|
||||
m.level = level
|
||||
m.rank = rank
|
||||
return m
|
||||
}
|
||||
|
||||
def getArroundAvgPrice(def list){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
if(prices){
|
||||
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
return avg_price;
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+280
@@ -0,0 +1,280 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
|
||||
class MatchEvaluateTest {
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据.xlsx"
|
||||
def total = 1500
|
||||
|
||||
static main(args) {
|
||||
def task = new MatchEvaluateTest()
|
||||
task.run()//小区匹配
|
||||
task.run1()//地图匹配
|
||||
// task.addDistrict();//新增人工处理小区,原文件匹配不上的去除
|
||||
task.addPrice()
|
||||
task.addArroundPrice()
|
||||
task.addAuthor()
|
||||
|
||||
// task.stat()
|
||||
}
|
||||
|
||||
def addArroundPrice(){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['A'] && !row['K']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['A'], row['B'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['K'] = priceObj.unitPrice
|
||||
row['L'] = (int) (priceObj.unitPrice * row['C'])
|
||||
row['O'] = "周边价格"
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def run(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['D'] && row['B']){
|
||||
def jsonObject = OpenApiClient.addressMatch5("重庆", row['B'], "property", false)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['D'] = jsonObject.get("id")
|
||||
row['E'] = jsonObject.get("city")
|
||||
row['F'] = jsonObject.get("district")
|
||||
row['G'] = jsonObject.get("name")
|
||||
row['H'] = jsonObject.get("address")
|
||||
row['I'] = jsonObject.get("propertyType")
|
||||
row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def run1(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['D'] && row['B']){
|
||||
def jsonObject = OpenApiClient.addressMatch("重庆", row['B'], null, true)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['D'] = jsonObject.get("id")
|
||||
row['E'] = jsonObject.get("city")
|
||||
row['F'] = jsonObject.get("district")
|
||||
row['G'] = jsonObject.get("name")
|
||||
row['H'] = jsonObject.get("address")
|
||||
row['I'] = jsonObject.get("propertyType")
|
||||
row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addPrice(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['J'] && row['E']){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(row['E'], row['F'], row['G'])
|
||||
if(priceObj){
|
||||
row['K'] = priceObj.avg_price
|
||||
row['L']= (int)(priceObj.avg_price * row['C'])
|
||||
}
|
||||
|
||||
|
||||
def avg_price = findAvgPrice(row['D'])
|
||||
if(row['K'] && avg_price){
|
||||
def weekendPrice = row['K'].toString().toDouble()
|
||||
def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price)
|
||||
row['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
def addAuthor(){
|
||||
// def path = "D:\\数据\\重庆农商行\\2022-05-06\\test.xlsx"
|
||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","高娟","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","徐翔翔","穆良书","沈燕芳"]
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['B']){
|
||||
if(index < avg){
|
||||
row['N'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
def addDistrict(){
|
||||
def path = "D:\\数据\\重庆农商行\\2022-05-06\\需新增小区.xlsx"
|
||||
def keys = new CopyOnWriteArrayList()
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { d->
|
||||
def flag = d['M']
|
||||
if(!flag){
|
||||
def city = d['E']
|
||||
def region = d['F']
|
||||
def newRegion = RegionData.formatRegion(city, region)
|
||||
if(newRegion){
|
||||
def name = d['G']
|
||||
name = nameFileter(name)
|
||||
|
||||
def formatAddress = null
|
||||
def address = d['H']
|
||||
if(address){
|
||||
formatAddress = AddressUtil.formatAddress(city, address)
|
||||
if(!formatAddress){
|
||||
if(address.contains("交叉口")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉口")+3)
|
||||
}else if(address.contains("交叉")){
|
||||
formatAddress = address.substring(0, address.toString().indexOf("交叉")+2)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Long glat = null
|
||||
Long glng = null
|
||||
Long blat = null
|
||||
Long blng = null
|
||||
if(d['K']){
|
||||
glat = d['L'].toString().toDouble() * 10000000000L
|
||||
glng = d['K'].toString().toDouble() * 10000000000L
|
||||
def baidu = BaiDuApiUtils.geoconv(d['K'].toString().toDouble(), d['L'].toString().toDouble(), 3)
|
||||
blng = baidu.x * 10000000000L
|
||||
blat = baidu.y * 10000000000L
|
||||
}
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
if(name.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(name.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}else{
|
||||
def type = d['I']
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
}else if(type.contains("商住")){
|
||||
estateType2 = "商住"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def key = city+"##"+newRegion+"##"+name
|
||||
if(name && !keys.contains(key)){
|
||||
def dist = CommonDatautils.findDistrict(city, newRegion, name, "district")
|
||||
if(!dist){
|
||||
keys << key
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",newRegion)
|
||||
data.put("name", name)
|
||||
data.put("estate_type2",estateType2)
|
||||
data.put("address", formatAddress)
|
||||
if(glat){
|
||||
data.put("lat",glat)
|
||||
data.put("lng",glng)
|
||||
data.put("lat2",blat)
|
||||
data.put("lng2",blng)
|
||||
}
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
index++
|
||||
DbUtils.seaweed.save(data);
|
||||
println index+" "+data
|
||||
addPrice(data)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+region +" 区域无法标准化"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-03-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-04-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
if(name.contains("(")){
|
||||
def delstr = name.substring(name.indexOf("("), name.indexOf(")")+1)
|
||||
name = name.replace(delstr,"")
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\2022-05-06\\模拟_测试数据_合并后 (1).xlsx",0,1)
|
||||
ExcelUtils.multiThreadingModify(path, { d ->
|
||||
if(d['B']){
|
||||
for(def row : datas){
|
||||
if(d['B']==row['B']){
|
||||
if(row['K']){
|
||||
d['P'] = row['K']
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,666 @@
|
||||
package com.fangjia.script.chongqing
|
||||
|
||||
import com.fangjia.core.util.SourceNameUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
|
||||
class PocTest {
|
||||
|
||||
def total = 3259
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new PocTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* A 序号
|
||||
* B 城市
|
||||
* C 区域
|
||||
* D 地址
|
||||
* E 面积
|
||||
* @return
|
||||
*/
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\重庆农商行\\poc\\test.xls"
|
||||
// match(inputfile)
|
||||
// addPrice(inputfile)
|
||||
// addArroundPrice(inputfile)
|
||||
// standard(inputfile)
|
||||
// addLoudongNum(inputfile)
|
||||
|
||||
|
||||
// addAuthor(inputfile)
|
||||
|
||||
// pkPrice(inputfile)
|
||||
|
||||
// addPpeValue(inputfile)
|
||||
|
||||
// matchMap(inputfile)
|
||||
// addDetail(inputfile, "D:\\数据\\重庆农商行\\poc\\楼盘详情.xlsx")
|
||||
//
|
||||
// createLoudong(inputfile,"D:\\数据\\重庆农商行\\poc\\底层数据-楼栋-户.xlsx")
|
||||
// createPaimai(inputfile,"D:\\数据\\重庆农商行\\poc\\法拍案例.xlsx")
|
||||
// createHouse(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||
// createDealPrice(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||
// createEvaluation(inputfile,"D:\\数据\\重庆农商行\\poc\\挂牌成交评估案例.xlsx")
|
||||
}
|
||||
|
||||
def addPpeValue(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(row['F'] && row['R']){
|
||||
def avg_price = findAvgPrice(row['R'])
|
||||
if(avg_price){
|
||||
def ppe15 = Math.abs((row['F'] - avg_price) * 100 / avg_price)
|
||||
row['Z'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, ppe15)
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def pkPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commid = row['R']
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
def jsonObject = OpenApiClient.webankEvaluate(dist.city, dist.region, dist.name, row['E'].toString())
|
||||
if(jsonObject != null && jsonObject.code==200) {
|
||||
def result = jsonObject.result
|
||||
if(result){
|
||||
row['Y'] = result.get("unitPrice")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def match(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['Q'] && row['D']){
|
||||
def city = row['B']
|
||||
def address = row['D']
|
||||
def region = row['C']
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['R'] = jsonObject.get("id")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchMap(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['Q'] && row['D']){
|
||||
def city = row['B']
|
||||
def address = row['D']
|
||||
def region = row['C']
|
||||
println "city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, true, region, null)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
def propertyType = jsonObject.get("propertyType")
|
||||
if(propertyType && (propertyType.contains("住宅") || propertyType.contains("别墅") || propertyType.contains("商住"))){
|
||||
row['Q'] = jsonObject.get("name")
|
||||
row['R'] = new ObjectId().toString()
|
||||
row['W'] = propertyType
|
||||
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(city, address)
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['F'] = priceObj.unitPrice
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commid = row['R']
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid.trim())
|
||||
if(dist){
|
||||
row['Q'] = dist.name
|
||||
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(priceObj){
|
||||
row['F'] = priceObj.avg_price
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addArroundPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']) {
|
||||
def arroundObj = FangjiaClientUtils.arroundPrice(row['B'], row['D'])
|
||||
if (arroundObj && arroundObj.msgCode == 200 && arroundObj.result) {
|
||||
def priceObj = arroundObj.result
|
||||
row['F'] = priceObj.unitPrice
|
||||
row['W'] = "周边均价"
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def standard(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def city = row['B']
|
||||
def address = row['D']
|
||||
if(address){
|
||||
println "[standard] address:"+address
|
||||
def jsonObject = OpenApiClient.standard2Action(city, address)
|
||||
if(jsonObject){
|
||||
println jsonObject
|
||||
row['G'] = jsonObject.get("province")
|
||||
row['H'] = jsonObject.get("city")
|
||||
row['I'] = jsonObject.get("district")
|
||||
row['J'] = jsonObject.get("township")
|
||||
row['K'] = jsonObject.get("street")
|
||||
row['L'] = jsonObject.get("streetNumber")
|
||||
row['M'] = jsonObject.get("buildingNumber")
|
||||
row['N'] = jsonObject.get("unitNumber")
|
||||
row['O'] = jsonObject.get("floor")
|
||||
row['P'] = jsonObject.get("roomNumber")
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def addLoudongNum(path){
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def commid = row['R']
|
||||
def houseNum = row['M']
|
||||
def unitNum = row['N']
|
||||
println "[addLoudongNum] commid:"+commid+" houseNum:"+houseNum
|
||||
if (commid && houseNum) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(commid, houseNum, unitNum)
|
||||
if (jsonObject && jsonObject.result) {
|
||||
if(jsonObject.result.size()>=1){
|
||||
def obj = jsonObject.result[0]
|
||||
row['T'] = obj.id
|
||||
row['S'] = obj.houseNumber
|
||||
}
|
||||
}
|
||||
|
||||
if(row['P']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(commid, houseNum, unitNum, row['P'])
|
||||
if (jsonObject1 && jsonObject1.result) {
|
||||
if(jsonObject1.result.size()>=1){
|
||||
def obj = jsonObject1.result[0]
|
||||
row['V'] = obj.id
|
||||
row['U'] = obj.roomNum
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def createLoudong(path, output){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
rets << ["输入城市","输入区域","输入地址","面积","楼栋号","楼栋ID","物业类型","主朝向","地上层数","地下层数","竣工日期","梯","户","总户数","景观","户号","户ID","主朝向","景观","户型结构"]
|
||||
for(def row : datas){
|
||||
def data = []
|
||||
data << row['B']
|
||||
data << row['C']
|
||||
data << row['D']
|
||||
data << row['E']
|
||||
if (row['R'] && row['M']) {
|
||||
def jsonObject = OpenApiClient.searchLoudong(row['R'], row['M'], row['N'])
|
||||
if (jsonObject && jsonObject.result && jsonObject.result.size()>=1) {
|
||||
def obj = jsonObject.result[0]
|
||||
data << obj.houseNumber
|
||||
data << obj.id
|
||||
data << obj.type// 物业类型
|
||||
data << ""//主朝向
|
||||
data << obj.floors//地上层数
|
||||
data << ""//地下层数
|
||||
data << obj.buildYear;//竣工日期
|
||||
data << ""//梯
|
||||
data << ""//户
|
||||
data << obj.households//总户数
|
||||
data << ""//景观
|
||||
}else{
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""// 物业类型
|
||||
data << ""//主朝向
|
||||
data << ""//地上层数
|
||||
data << ""//地下层数
|
||||
data << ""//竣工日期
|
||||
data << ""//梯
|
||||
data << ""//户
|
||||
data << ""//总户数
|
||||
data << ""//景观
|
||||
}
|
||||
|
||||
if(row['P']){
|
||||
def jsonObject1 = OpenApiClient.searchHouseLoudong(row['R'], row['M'], row['N'], row['P'])
|
||||
if (jsonObject1 && jsonObject1.result) {
|
||||
if(jsonObject1.result.size()>=1){
|
||||
def obj = jsonObject1.result[0]
|
||||
|
||||
data << obj.roomNum//户号
|
||||
data << obj.id//户ID
|
||||
data << obj.direction//主
|
||||
data << obj.sight//景观
|
||||
if(obj.roomNum && obj.roomNum.contains("跃")){
|
||||
data << "跃式"//户型结构
|
||||
}else{
|
||||
data << ""//户型结构
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write(output, rets, "底层数据-楼栋-户")
|
||||
}
|
||||
|
||||
def addDetail(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["输入城市","输入区域","输入地址","小区名称","小区ID","小区别名","小区地址","板块","街道","基准价","经纬度","竣工时间","开发商名称","绿化率","容积率","车位个数","物业公司","物业费","住宅幢数","总户数","车位配比","占地面积","是否学区","周边教育","周边交通","周边商业","周边医院","周边金融机构"]
|
||||
for(def d : datas){
|
||||
def commid = d['R']
|
||||
println "[addDetail] commid:"+commid
|
||||
if(commid){
|
||||
def dist = CommonDatautils.findDistrictById(commid)
|
||||
if(dist){
|
||||
def row = []
|
||||
def jsonObject = OpenApiClient.districtDetail(dist.city, dist.region, dist.name)
|
||||
if(jsonObject) {
|
||||
|
||||
row << d['B']
|
||||
row << d['C']
|
||||
row << d['D']
|
||||
row << jsonObject.get("name")//小区名称
|
||||
row << commid//小区ID
|
||||
if(jsonObject.get("alias")){
|
||||
row << jsonObject.get("alias").join(",").replace('"',"")//小区别名
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << jsonObject.get("address")//小区地址
|
||||
row << jsonObject.get("commercialArea")//板块
|
||||
row << jsonObject.get("township")//街道***********
|
||||
row << jsonObject.get("avgPrice")//基准价
|
||||
row << jsonObject.get("lng").toString()+","+jsonObject.get("lat").toString()//经纬度
|
||||
row << jsonObject.get("finishBuildTime")//竣工时间
|
||||
row << jsonObject.get("developer")//开发商名称
|
||||
row << SeaweedUtils.formatVirescencePercent(jsonObject.get("greenPercentage"))//绿化率
|
||||
row << SeaweedUtils.formatCubagePercent(jsonObject.get("plotRatio"))//容积率
|
||||
row << SeaweedUtils.formatCarSiteCount(jsonObject.get("parking"))//车位个数
|
||||
row << jsonObject.get("propertyManagement")//物业公司
|
||||
row << jsonObject.get("managementFee")//物业费
|
||||
row << jsonObject.get("buildingCount")//住宅幢数
|
||||
row << SeaweedUtils.formatHouseholdCount(jsonObject.get("totalUnitCount"))//总户数
|
||||
row << SeaweedUtils.calcCarAndHouseholdRadio(jsonObject.get("parking"), jsonObject.get("totalUnitCount"))//车位配比*********
|
||||
row << SeaweedUtils.formatAllArea(jsonObject.get("floorSpace"))//占地面积
|
||||
|
||||
}
|
||||
|
||||
def facilityObj = OpenApiClient.facilitys(dist.city, dist.region, dist.name)
|
||||
if(facilityObj && facilityObj.result){
|
||||
def result = facilityObj.result
|
||||
if(result.keySchool){
|
||||
row << '是'
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.school){
|
||||
row << result.school.join(",").replace('"',"")
|
||||
}else {
|
||||
row << ""
|
||||
}
|
||||
|
||||
def bus = []
|
||||
if(result.bus){
|
||||
bus.addAll(result.bus)
|
||||
}
|
||||
if(result.metro){
|
||||
bus.addAll(result.metro)
|
||||
}
|
||||
|
||||
if(bus){
|
||||
row << bus.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.supermarket){
|
||||
row << result.supermarket.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.hospital){
|
||||
row << result.hospital.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(result.bank){
|
||||
row << result.bank.join(",").replace('"',"")
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
rows << row
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "底层数据-小区")
|
||||
}
|
||||
|
||||
def createPaimai(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["输入城市","输入区域","输入地址","小区名称","标的名称","楼栋","户号","面积","案例时间","类型","状态","成交单价","评估单价"]
|
||||
def keys = []
|
||||
for(def d : datas){
|
||||
def commid = d['R']
|
||||
println "[createPaimai] commid:"+commid
|
||||
if(commid) {
|
||||
def dist = CommonDatautils.findDistrictById(commid)
|
||||
if (dist) {
|
||||
def key = dist.city+"#"+dist.region+"#"+dist.name
|
||||
if(keys.contains(key)){
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.searchPaimais(dist.city, dist.region, dist.name,"20220217",100)
|
||||
if(jsonObject && jsonObject.result){
|
||||
def results = jsonObject.result
|
||||
for(def r: results){
|
||||
def row = []
|
||||
row << d['B']
|
||||
row << d['C']
|
||||
row << d['D']
|
||||
row << r.matchName
|
||||
row << r.address
|
||||
row << r.houseNum
|
||||
row << r.roomNum
|
||||
row << r.area
|
||||
if(r.startDate){
|
||||
row << r.startDate.substring(0, r.startDate.indexOf("T"))
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << r.count
|
||||
row << r.status
|
||||
row << r.tradeAvgPrice
|
||||
row << r.tradePrice
|
||||
row << r.id
|
||||
rows << row
|
||||
}
|
||||
println dist.name +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "拍卖案例")
|
||||
}
|
||||
|
||||
def createHouse(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
def commid = d['R']
|
||||
// println "[createHouse] commid:"+commid
|
||||
if (commid) {
|
||||
def key = d['Q']
|
||||
if (keys.contains(key)) {
|
||||
continue
|
||||
}
|
||||
keys << key
|
||||
def jsonObject = OpenApiClient.houseList(commid,"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "挂牌"
|
||||
row << r.districtName
|
||||
if(r.estateType){
|
||||
row << r.estateType
|
||||
}else{
|
||||
if(r.purpose){
|
||||
row << r.purpose
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
}
|
||||
|
||||
if(r.room){
|
||||
def roomStr = r.room+"室"
|
||||
if(r.hall){
|
||||
roomStr = roomStr +r.hall+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.area
|
||||
row << r.fitment
|
||||
row << r.pDate
|
||||
row << ""
|
||||
row << r.avgPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.height){
|
||||
row << r.height
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << r.direction
|
||||
row << SourceNameUtil.getResourceName(r.root)
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['R'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "挂牌案例")
|
||||
}
|
||||
|
||||
def createEvaluation(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
def commid= d['R']
|
||||
println "[createEvaluation] commid:"+commid
|
||||
if (commid) {
|
||||
if (keys.contains(commid)) {
|
||||
continue
|
||||
}
|
||||
keys << commid
|
||||
def jsonObject = OpenApiClient.evaluaionList(commid,"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "评估"
|
||||
row << r.commName
|
||||
row << ""
|
||||
|
||||
if(r.roomcnt){
|
||||
def roomStr = r.roomcnt+"室"
|
||||
if(r.livingroomcnt){
|
||||
roomStr = roomStr +r.livingroomcnt+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.buildArea
|
||||
row << r.fitment
|
||||
row << r.cDate.substring(0, r.cDate.indexOf("T"))
|
||||
row << ""
|
||||
row << r.unitPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.floorcnt){
|
||||
row << r.floorcnt
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.houseaspect && !r.houseaspect.contains("暂")){
|
||||
row << r.houseaspect
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << "房价网"
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['R'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "评估案例")
|
||||
}
|
||||
|
||||
|
||||
def createDealPrice(path, output){
|
||||
def datas = ExcelUtils.read(path, 0, 1)
|
||||
def rows = []
|
||||
rows << ["案例类型","小区名称","物业类型","户型","面积","装修","案例时间","成交周期","单价","总价","当前层","总楼层","朝向","来源"]
|
||||
def keys = []
|
||||
for(def d : datas) {
|
||||
def commid= d['R']
|
||||
println "[createDealPrice] commid:"+commid
|
||||
if (commid) {
|
||||
if (keys.contains(commid)) {
|
||||
continue
|
||||
}
|
||||
keys << commid
|
||||
def jsonObject = OpenApiClient.searchDealPrice(commid,"20210501",20)
|
||||
if(jsonObject && jsonObject.result) {
|
||||
def results = jsonObject.result
|
||||
for (def r : results) {
|
||||
def row = []
|
||||
row << "成交"
|
||||
row << r.districtName
|
||||
if(r.type){
|
||||
row << r.type
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.room){
|
||||
def roomStr = r.room+"室"
|
||||
if(r.hall){
|
||||
roomStr = roomStr +r.hall+"厅"
|
||||
}
|
||||
row << roomStr
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
row << r.area
|
||||
row << r.fitment
|
||||
row << r.tradeDate.substring(0, r.tradeDate.indexOf("T"))
|
||||
row << ""
|
||||
row << r.avgPrice
|
||||
row << r.totalPrice
|
||||
if(r.floor){
|
||||
row << r.floor
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
if(r.height){
|
||||
row << r.height
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
if(r.direction && !r.direction.contains("暂")){
|
||||
row << r.direction
|
||||
}else{
|
||||
row << ""
|
||||
}
|
||||
|
||||
row << r.source
|
||||
rows << row
|
||||
}
|
||||
|
||||
println d['R'] +" "+results.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write(output, rows, "成交案例")
|
||||
}
|
||||
|
||||
def addAuthor(def path){
|
||||
def authors = ["丁世菁","王文艳","董彦均","曹玉彦","姚静梦","付贵生","赵灵芝","沈健","许镕杰","徐鹤","穆良书","沈燕芳"]
|
||||
def personTotal = authors.size()
|
||||
def avg = Math.ceil(total/personTotal)
|
||||
|
||||
def index = 0
|
||||
def count = 0;
|
||||
ExcelUtils.modify(path, { row ->
|
||||
if(row['B']){
|
||||
if(index < avg){
|
||||
row['X'] = authors[count]
|
||||
}
|
||||
index++
|
||||
if(index >= avg){
|
||||
count++
|
||||
index = 0
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def findAvgPrice(def commid){
|
||||
def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2";
|
||||
def avg_price = null
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
avg_price = r.target_avg_price
|
||||
});
|
||||
return avg_price
|
||||
}
|
||||
}
|
||||
+94
-16
@@ -68,6 +68,10 @@ class CommonDatautils {
|
||||
|
||||
public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl)
|
||||
|
||||
public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", mongoDbUrl)
|
||||
public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", mongoDbUrl)
|
||||
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
return house_dataStoreConnection
|
||||
@@ -586,8 +590,8 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrict(String city, String region, String name, String cat){
|
||||
|
||||
public static def findDistrictCount(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
if(region){
|
||||
@@ -598,7 +602,24 @@ class CommonDatautils {
|
||||
q.put("cat", cat)
|
||||
q.put("status", 0)
|
||||
|
||||
|
||||
def count = getDataStoreConnection().count(q);
|
||||
return count;
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrict(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("city", city);
|
||||
if(region){
|
||||
q.put("region", region);
|
||||
}
|
||||
|
||||
q.put("name", name);
|
||||
q.put("cat", cat)
|
||||
q.put("status", 0)
|
||||
|
||||
|
||||
DSCursor cursor = getDataStoreConnection().find(q);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
@@ -609,6 +630,7 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def findDistrictList(String city, String region, String name, String cat){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
@@ -704,7 +726,7 @@ class CommonDatautils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","ÎâÖз9ÅªÐ¡Çø")
|
||||
// q.put("name","ÉÍû·¼²ÝÂ̰¶")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -827,15 +849,24 @@ class CommonDatautils {
|
||||
keys.put("out_pics", 1);
|
||||
keys.put("lat", 1);
|
||||
keys.put("lng", 1);
|
||||
keys.put("lat2", 1);
|
||||
keys.put("lng2", 1);
|
||||
keys.put("name_cat", 1);
|
||||
keys.put("finish_building_date", 1)
|
||||
keys.put("mapping_alias", 1)
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
keys.put("sid", 1)
|
||||
try{
|
||||
DSCursor cursor = getDataStoreConnection().find(q, keys);
|
||||
if(cursor){
|
||||
for(def obj : cursor){
|
||||
return obj;
|
||||
}
|
||||
}
|
||||
}catch(Exception e){
|
||||
println "error_id:"+id
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -1052,9 +1083,16 @@ class CommonDatautils {
|
||||
|
||||
|
||||
public static def queryDistrictById(def id){
|
||||
def dbo = null
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("_id", new ObjectId(id));
|
||||
def dbo = getDataStoreConnection().findOne(q);
|
||||
try{
|
||||
q.put("_id", new ObjectId(id.trim().toString()));
|
||||
dbo = getDataStoreConnection().findOne(q);
|
||||
}catch(Exception e){
|
||||
println id
|
||||
}
|
||||
|
||||
|
||||
return dbo;
|
||||
}
|
||||
|
||||
@@ -2384,6 +2422,7 @@ class CommonDatautils {
|
||||
q.put("block", block);
|
||||
}
|
||||
q.put("cat", cat);
|
||||
q.put("status",0)
|
||||
def cur = getDataStoreConnection().find(q);
|
||||
return cur.count();
|
||||
}
|
||||
@@ -3040,18 +3079,26 @@ class CommonDatautils {
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)).limit(1)
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
DBObject dbo;
|
||||
def prices = []
|
||||
def index = 0
|
||||
while(cur.hasNext()) {
|
||||
dbo = cur.next()
|
||||
def price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
def obj = cur.next()
|
||||
if(index == 0){
|
||||
dbo = obj
|
||||
}
|
||||
prices << obj.get("avg_price");
|
||||
index++
|
||||
}
|
||||
|
||||
return dbo;
|
||||
if(prices){
|
||||
def avg_price = (int)(prices.sum()/prices.size())
|
||||
dbo.avg_price = avg_price
|
||||
return dbo;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
@@ -3166,5 +3213,36 @@ class CommonDatautils {
|
||||
}
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
static def findCityAvgRentRadio(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
|
||||
def dbcursor = SQMETRE_RENT_COLL.find(q)
|
||||
def avg_price = null
|
||||
def radios = []
|
||||
for(def d : dbcursor){
|
||||
if(d.price_to_rent){
|
||||
radios << d.price_to_rent.toInteger()
|
||||
}
|
||||
}
|
||||
def avg_radios = (int)(radios.sum()/radios.size())
|
||||
return avg_radios;
|
||||
}
|
||||
|
||||
static def findLoudong(commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("id_in_seaweed", commid)
|
||||
def count = LOUDONG_LINE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
static def findHouseLoudong(commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("district_id", commid)
|
||||
def count = HOUSE_LOUDONG_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+3
-3
@@ -16,9 +16,7 @@ import org.apache.commons.lang.math.RandomUtils;
|
||||
|
||||
class DataHandleUtils {
|
||||
|
||||
static main(args){
|
||||
|
||||
}
|
||||
|
||||
|
||||
static def getStandardRegion(def regionMap, def region){
|
||||
def newRegion = null;
|
||||
@@ -94,6 +92,8 @@ class DataHandleUtils {
|
||||
BigDecimal deSource = new BigDecimal(dSource);
|
||||
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def trimChar(String c) {
|
||||
try {
|
||||
|
||||
+1
-1
@@ -35,7 +35,7 @@ class DistrictAroundFacilitiesBatch extends BaseBatch {
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def cities = ["北京", "上海"]
|
||||
def cities = ["北京", "上海"]//
|
||||
for (def city : cities) {
|
||||
def ret = CommonDatautils.getAllDistricts("district", city)
|
||||
// def ret = getAllDistrictBaseInfo("上海", '闵行', '万源城', "district");
|
||||
|
||||
+86
-5
@@ -6,10 +6,16 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.jusfoun.JusfounCommon
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.DecimalFormat
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
@@ -19,8 +25,11 @@ import java.util.concurrent.TimeUnit
|
||||
class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonthDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def charset = "GBK"
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictPriceExportBatch(), args)
|
||||
}
|
||||
@@ -32,6 +41,9 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
def ret = createDatas("¸£ÖÝ")
|
||||
|
||||
saveToDB(ret)
|
||||
|
||||
|
||||
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
|
||||
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
|
||||
if(CommonDatautils.isWindow()){
|
||||
@@ -76,10 +88,10 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
|
||||
def createDatas(def city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", null);
|
||||
|
||||
def lastDate = new Date()
|
||||
def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1)
|
||||
|
||||
def endDateStr = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-10"
|
||||
def dataStr = DateUtils.add(endDateStr, -30)
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD(dataStr)
|
||||
def lastDate = DateUtils.parseFromTightByPattern(endDateStr, "yyyy-MM-dd")
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_202111.csv"))
|
||||
// def map2 = [:]
|
||||
// for(def s : list2) {
|
||||
@@ -88,17 +100,34 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
// map2.put(key, row[5])
|
||||
// }
|
||||
|
||||
def lastMonthPriceMap = findFzDistrictStat("¸£ÖÝ",lastMonthDate)
|
||||
def result = []
|
||||
for(def rMap : districts){
|
||||
|
||||
def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate)
|
||||
if(dbo){
|
||||
def lastMonthPrice = lastMonthPriceMap.get(rMap.get("_id").toString())
|
||||
def curPrice = dbo.get("avg_price")
|
||||
if(lastMonthPrice && curPrice){
|
||||
def markup = (curPrice - lastMonthPrice)*100/lastMonthPrice
|
||||
if(Math.abs(markup)>3){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(markup < 0){
|
||||
markup = randMarkup*(-1)
|
||||
}else{
|
||||
markup = randMarkup
|
||||
}
|
||||
curPrice = (int)(lastMonthPrice*markup/100 + lastMonthPrice)
|
||||
}
|
||||
}
|
||||
|
||||
def m = []
|
||||
m << dbo.get("_id").toString()//¾ù¼ÛID
|
||||
m << rMap.get("_id").toString()//Â¥ÅÌID
|
||||
m << rMap.city
|
||||
m << rMap.region
|
||||
m << rMap.name
|
||||
m << dbo.get("avg_price")
|
||||
m << curPrice
|
||||
// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name)
|
||||
// if(lastMonthPrice && dbo.get("avg_price")){
|
||||
// lastMonthPrice = lastMonthPrice.toInteger()
|
||||
@@ -125,6 +154,19 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
def findFzDistrictStat(city,lastMonthDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("month", lastMonthDate)
|
||||
def dbcursor = FZ_DISTRICT_STAT_COLL.find(q)
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
priceMap.put(d.comm_id, d.avg_price)
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
def export(def fileName, def datas){
|
||||
def headers = ["id","comm_id","city","district","name","avg_price","year_month"]
|
||||
writeCSV(headers, fileName, datas);
|
||||
@@ -152,5 +194,44 @@ class DistrictPriceExportBatch extends BaseBatch{
|
||||
file.append(System.getProperty("line.separator"));
|
||||
}
|
||||
}
|
||||
|
||||
//Ëæ»ú¸¡µãÊý
|
||||
public static double nextDouble(final double min, final double max) throws Exception {
|
||||
|
||||
DecimalFormat df = new DecimalFormat("#.00");
|
||||
if (max < min) {
|
||||
throw new Exception("min < max");
|
||||
}
|
||||
if (min == max) {
|
||||
return min;
|
||||
}
|
||||
return Double.parseDouble(df.format(min + ((max - min) * new Random().nextDouble())));
|
||||
}
|
||||
|
||||
def saveToDB(rets){
|
||||
for(def r : rets){
|
||||
def q = new BasicDBObject()
|
||||
q.put("month", curDate.toInteger())
|
||||
q.put("comm_id", r[1])
|
||||
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("avg_price", r[5])
|
||||
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
def o = new BasicDBObject()
|
||||
o.put("_id", new ObjectId(r[0]))
|
||||
o.put("comm_id", r[1])
|
||||
o.put("city", r[2])
|
||||
o.put("region", r[3])
|
||||
o.put("name", r[4])
|
||||
o.put("avg_price", r[5].toInteger())
|
||||
o.put("month", r[6].toInteger())
|
||||
FZ_DISTRICT_STAT_COLL.save(o)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+73
-7
@@ -8,7 +8,9 @@ import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 福州数据结果统计
|
||||
@@ -17,16 +19,42 @@ class FzDataResultStat {
|
||||
|
||||
public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
}
|
||||
|
||||
def margeExcel(){
|
||||
|
||||
def list2= ExcelUtils.read("D:\\数据\\福州\\价格比对_202204_新2.xlsx", 0 ,1)
|
||||
|
||||
def map2 = [:]
|
||||
for(def s : list2){
|
||||
def key = s["A"]+"##"+s["B"]+"##"+s["C"]
|
||||
map2.put(key, s["D"])
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\福州\\价格比对_202204_新3.xlsx",{row->
|
||||
if(row["A"]){
|
||||
def key = row["A"]+"##"+row["B"]+"##"+row["C"]
|
||||
def price = map2.get(key)
|
||||
if(price){
|
||||
row['L'] = price
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
|
||||
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
|
||||
def map1 = [:]
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
@@ -137,16 +165,13 @@ class FzDataResultStat {
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate));
|
||||
DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
|
||||
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def price = 0;
|
||||
def prices = [];
|
||||
while(cur.hasNext()) {
|
||||
DBObject dbo = cur.next()
|
||||
price = dbo.get("avg_price");
|
||||
if(price){
|
||||
break;
|
||||
}
|
||||
prices << dbo.get("avg_price");
|
||||
}
|
||||
|
||||
return price;
|
||||
def avg_price = (int)(prices.sum()/prices.size())
|
||||
return avg_price;
|
||||
}
|
||||
|
||||
|
||||
@@ -165,4 +190,45 @@ class FzDataResultStat {
|
||||
weekends = weekends.reverse();
|
||||
return weekends;
|
||||
}
|
||||
|
||||
|
||||
def saveCsvData(){
|
||||
def files = getAllFiles("C:\\Users\\gaojuan\\福州数据(1)")
|
||||
if(files){
|
||||
for(def filepath : files){
|
||||
List<String> list1= FileUtils.readLines(filepath)
|
||||
println filepath
|
||||
for(def s : list1){
|
||||
def row = s.split("@\\|@")
|
||||
def o = new BasicDBObject()
|
||||
o.put("_id", new ObjectId(row[0]))
|
||||
o.put("comm_id", row[1])
|
||||
o.put("city", row[2])
|
||||
o.put("region", row[3])
|
||||
o.put("name", row[4])
|
||||
o.put("avg_price", row[5].toInteger())
|
||||
o.put("month", row[6].toInteger())
|
||||
FZ_DISTRICT_STAT_COLL.save(o)
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def static getAllFiles(def path) {
|
||||
|
||||
File file = new File(path);
|
||||
File[] tempList = file.listFiles();
|
||||
|
||||
def files = [];
|
||||
for (int i = 0; i < tempList.length; i++) {
|
||||
if (tempList[i].isFile()) {
|
||||
files << tempList[i];
|
||||
}
|
||||
if (tempList[i].isDirectory()) {
|
||||
System.out.println("文件夹:" + tempList[i]);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202204.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+17
-11
@@ -112,22 +112,28 @@ class TradeExportBatch extends BaseBatch{
|
||||
// q.put("trade_date", new BasicDBObject(QueryOperators.LT, endDate))
|
||||
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(1000);
|
||||
def rets = []
|
||||
def keys = []
|
||||
if (cursor != null) {
|
||||
while (cursor.hasNext()) {
|
||||
DBObject dbo = cursor.next();
|
||||
def district = districtMap.get(dbo.city + "_" + dbo.region + "_" + dbo.district_name);
|
||||
if(district){
|
||||
def ret = [];
|
||||
ret << dbo._id.toString()
|
||||
ret << district._id.toString()
|
||||
ret << dbo.city
|
||||
ret << dbo.region
|
||||
ret << dbo.district_name
|
||||
ret << dbo.avg_price
|
||||
ret << dbo.area
|
||||
ret << DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
|
||||
ret << dbo.total_price
|
||||
rets << ret;
|
||||
def tradeDateStr = DateUtils.formatDateByPattern(dbo.trade_date, "yyyy-MM-dd")
|
||||
def key = district._id.toString() +"##" + dbo.avg_price +"##"+dbo.area+"##"+tradeDateStr
|
||||
if(!keys.contains(key)){
|
||||
keys << key
|
||||
def ret = [];
|
||||
ret << dbo._id.toString()
|
||||
ret << district._id.toString()
|
||||
ret << dbo.city
|
||||
ret << dbo.region
|
||||
ret << dbo.district_name
|
||||
ret << dbo.avg_price
|
||||
ret << dbo.area
|
||||
ret << tradeDateStr
|
||||
ret << dbo.total_price
|
||||
rets << ret;
|
||||
}
|
||||
}
|
||||
}
|
||||
println "³É½»×ܼƣº"+rets.size()
|
||||
|
||||
+12
-8
@@ -35,7 +35,8 @@ class EsHousePriceStat extends BaseBatch{
|
||||
def run(){
|
||||
def cityMap = CommonDatautils.getAllCityRegions()
|
||||
cityMap.each{city,regions->
|
||||
findPriceByCity(city, regions);
|
||||
|
||||
findPriceByCity(city, regions);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,9 +65,9 @@ class EsHousePriceStat extends BaseBatch{
|
||||
q.put("region" , r.region)
|
||||
q.put("month", r.month)
|
||||
def mom = null
|
||||
if(lastObj){
|
||||
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
|
||||
mom = DataHandleUtils.getDoubleRound(4, mom)
|
||||
if(lastObj && lastObj.avg_price && r.price){
|
||||
mom = (r.price -lastObj.avg_price)/lastObj.avg_price
|
||||
mom = DataHandleUtils.getDoubleRound(4, mom)
|
||||
}
|
||||
if(obj){
|
||||
if(obj.avg_price != r.price){
|
||||
@@ -103,7 +104,7 @@ class EsHousePriceStat extends BaseBatch{
|
||||
|
||||
def findEsAvgPrice(city, region){
|
||||
def detail = [:]
|
||||
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 1])
|
||||
detail.put("date_histogram",[ "field": "publish_date","interval": "1M","time_zone": "Asia/Shanghai","min_doc_count": 0])
|
||||
detail.put("aggs",[ "1": ["moving_avg": ["buckets_path": "1-metric"]],"1-metric": ["avg": ["field": "avg_price"]]])
|
||||
|
||||
def must = []
|
||||
@@ -143,15 +144,18 @@ class EsHousePriceStat extends BaseBatch{
|
||||
if(!priceValue){
|
||||
priceValue = row['1-metric']
|
||||
}
|
||||
if(priceValue){
|
||||
if(priceValue && priceValue.value){
|
||||
price = priceValue.value.toInteger()
|
||||
}
|
||||
result.price = price;
|
||||
def date = row.key_as_string.replace("-","").substring(0, 6)
|
||||
def curMonth = DateUtils.formatDateByPattern(new Date(),"yyyyMM")
|
||||
if(date!=curMonth){
|
||||
result.month = date.toInteger()
|
||||
rets << result
|
||||
if(date.toInteger()>200001 && result.price){
|
||||
result.month = date.toInteger()
|
||||
rets << result
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,7 @@ import org.apache.commons.lang.StringUtils
|
||||
|
||||
class OpenApiUtils {
|
||||
static def matchDistrict(def city, def address, def suggestDistrict, def preciseLevelflag){
|
||||
address = address.replace("£¨","(").replace("£©",")").replace(" ","")
|
||||
address = address.replace("£¨","(").replace("£©",")").replace(" ","").replace("#","").replace("<","").replace(">","").replace("%","").replace("\"","")
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&suggestDistrict="+suggestDistrict+"&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district&__refresh_cache=1"
|
||||
if(preciseLevelflag){
|
||||
url = url +"&preciseLevel=strict"
|
||||
@@ -164,6 +164,7 @@ class OpenApiUtils {
|
||||
if(StringUtils.isEmpty(city) || StringUtils.isEmpty(address)){
|
||||
return null;
|
||||
}
|
||||
address = address.trim().replace(" ","")
|
||||
String url = "https://restapi.amap.com/v3/place/text?keywords=" + address + "&types=&city=" + city + "&output=json&offset=20&page=1&key=2af0b8533415a612b98d59bb93d664b9&extensions=all"
|
||||
|
||||
List<com.alibaba.fastjson.JSONObject> poiList = null;
|
||||
|
||||
+1
-1
@@ -48,7 +48,7 @@ class VagueAddressLocationBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
//"北京","常州",
|
||||
// def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity();
|
||||
def citys =["±±¾©"] //CommonDatautils.findAllCity();
|
||||
def citys =["ÖØÇì"] //CommonDatautils.findAllCity();
|
||||
//,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄"
|
||||
//市谊大厦 华三新村 阳光家园
|
||||
for (def city : citys) {
|
||||
|
||||
+4
-4
@@ -42,17 +42,17 @@ class YouDaDataParser {
|
||||
static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search"
|
||||
|
||||
|
||||
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc"
|
||||
static def esf_targetUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_doc"
|
||||
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/doc/_search"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2022-02-01"
|
||||
static def statDate = "2022-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser();//友达二手房存入mongo
|
||||
task.esfParser();//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2022年4月\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.fangjia.script.gj.apiLog
|
||||
|
||||
import com.alibaba.excel.util.DateUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.Calendar;
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ApiTimeOutAnalyBatch extends BaseBatch{
|
||||
|
||||
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ApiTimeOutAnalyBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def uriCountMap = findTimeOutUri()
|
||||
sendMessage(uriCountMap)
|
||||
}
|
||||
|
||||
def sendMessage(uriCountMap){
|
||||
def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:"
|
||||
def total = message.length()
|
||||
uriCountMap.each{uri,count->
|
||||
if(count>=5){
|
||||
message = message + uri +"\n"
|
||||
}
|
||||
}
|
||||
if(message.length()>total){
|
||||
println message
|
||||
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, message)
|
||||
}
|
||||
}
|
||||
|
||||
def findTimeOutUri(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("time", new BasicDBObject('$gte', 3000))
|
||||
def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5)
|
||||
q.put("c_date", new BasicDBObject('$gte', startTime))
|
||||
def dbcursor = LOG.find(q)
|
||||
def uriCountMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def count = uriCountMap.get(d.request_uri)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
uriCountMap.put(d.request_uri, count + 1)
|
||||
}
|
||||
return uriCountMap
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package com.fangjia.script.gj.apiLog
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ApiUserCountAnalyBatch extends BaseBatch{
|
||||
|
||||
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
|
||||
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ApiUserCountAnalyBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def rets = findAllUsers()
|
||||
sendMessage(rets)
|
||||
}
|
||||
|
||||
|
||||
def sendMessage(def rets){
|
||||
def message = "【Api接口使用提醒】\n"
|
||||
def total = message.length()
|
||||
for(def r : rets){
|
||||
def type = "正式帐户"
|
||||
if(r.type!=0){
|
||||
type = "测试帐户"
|
||||
}
|
||||
message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n"
|
||||
}
|
||||
if(message.length()>total){
|
||||
println message
|
||||
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, message)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findAllUsers(){
|
||||
def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'"
|
||||
def rows = dbFangjia.rows(sql)
|
||||
def rets = []
|
||||
for(def row : rows){
|
||||
def obj = findResources(row.id)
|
||||
if(obj){
|
||||
def map = obj.total_count_map
|
||||
def uris = []
|
||||
map.each{k,v->
|
||||
//def visit_count = findVisitCount(row.id, k)
|
||||
def total_visit_count = findAllVisitCount(row.id, k)
|
||||
println row.id +" "+k+" "+v+" "+total_visit_count
|
||||
if(v && total_visit_count){
|
||||
def precent = v*0.8;
|
||||
if(total_visit_count>=precent){
|
||||
uris << k
|
||||
}
|
||||
}
|
||||
}
|
||||
if(uris){
|
||||
rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllVisitCount(userId, uri){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId.toString())
|
||||
q.put("uri", uri)
|
||||
q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date())))
|
||||
def dbcursor = VISIT_COLL.find(q)
|
||||
def total = 0
|
||||
for(def obj : dbcursor){
|
||||
total = total +obj.visit_count
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
def findVisitCount(userId, uri){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId.toString())
|
||||
q.put("uri", uri)
|
||||
q.put("visit_date", DateUtils.truncateDay(new Date()))
|
||||
def obj = VISIT_COLL.findOne(q)
|
||||
if(obj){
|
||||
return obj.visit_count
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findResources(def userId){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId)
|
||||
return AUTHORITY_COLL.findOne(q)
|
||||
}
|
||||
}
|
||||
+44
-26
@@ -18,44 +18,62 @@ class CcbCityPriceStat {
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
def citys = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def citys = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","临汾","聊城","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","齐齐哈尔","眉山","郴州","营口","蚌埠","赤峰","大庆","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","朔州","随州","大理","通化","楚雄","巴彦淖尔","乌兰察布","松原","景德镇","新余","辽阳","阜新","黔南","酒泉","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","钦州","内江","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","鹤岗","萍乡","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","湘西","昭通","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
def rets = []
|
||||
def startDate = DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-06-01","yyyy-MM-dd")
|
||||
|
||||
for(def city : citys){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2021-05-30"))
|
||||
def cityRets = []
|
||||
for(def dist : districts){
|
||||
def tradeAvgPrice = findTrades(dist, "2021-02-01", "2021-06-01")
|
||||
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
|
||||
if(tradeAvgPrice && weekendPrice){
|
||||
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
|
||||
cityRets << row
|
||||
println row.toString()
|
||||
def matchTotalCount = findMatchTradeCount(city, startDate, endDate)
|
||||
println city +" "+matchTotalCount
|
||||
if(matchTotalCount){
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, DateUtils.parseFromTightYYYYMMDD("2022-05-29"))
|
||||
def cityRets = []
|
||||
for(def dist : districts){
|
||||
def tradeAvgPrice = findTrades(dist, "2022-02-01", "2022-06-01")
|
||||
def weekendPrice = priceMap.get(city +"##"+dist.region+"##"+dist.name)
|
||||
if(tradeAvgPrice && weekendPrice){
|
||||
def row = [city, dist.region, dist.name, weekendPrice, tradeAvgPrice.toInteger()]
|
||||
cityRets << row
|
||||
println row.toString()
|
||||
}
|
||||
}
|
||||
print city +" "+ cityRets.size()
|
||||
rets.addAll(cityRets)
|
||||
}
|
||||
print city +" "+ cityRets.size()
|
||||
rets.addAll(cityRets)
|
||||
|
||||
}
|
||||
|
||||
println "×ÜÁ¿£º"+rets.size()
|
||||
ExcelUtils.write("d:\\数据\\92个城市价格数据.xlsx", rets, "所有")
|
||||
ExcelUtils.write("d:\\数据\\234个城市价格数据.xlsx", rets, "所有")
|
||||
}
|
||||
|
||||
|
||||
def findMatchTradeCount(city, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', true))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
// q.put("city", dist.city)
|
||||
// q.put("region", dist.region)
|
||||
//
|
||||
// if(dist.alias){
|
||||
// if(!dist.alias.contains(dist.name)){
|
||||
// dist.alias << dist.name
|
||||
// }
|
||||
// q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
// }else{
|
||||
// q.put("district_name", dist.name)
|
||||
// }
|
||||
q.put("match_id", dist._id.toString())
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
|
||||
+6
-6
@@ -199,18 +199,18 @@ class CcbCodeUtils {
|
||||
static def getBuildStructureCode(def buildStructure){
|
||||
if(buildStructure){
|
||||
if(buildStructure.contains("钢结构")){
|
||||
return "001";
|
||||
return "3";
|
||||
}else if(buildStructure.contains("钢、钢筋混凝土结构")){
|
||||
return "002";
|
||||
return "1";
|
||||
}else if(buildStructure.contains("钢筋混凝土结构")){
|
||||
return "003";
|
||||
return "1";
|
||||
}else if(buildStructure.contains("砖木结构")){
|
||||
return "004";
|
||||
return "10";
|
||||
}else if(buildStructure.contains("混合结构")){
|
||||
return "005";
|
||||
return "2";
|
||||
}
|
||||
}
|
||||
return "006";
|
||||
return "11";
|
||||
}
|
||||
|
||||
static def getDistHouseTypeCode(def estateType){
|
||||
|
||||
@@ -50,4 +50,32 @@ class FileUtils {
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
|
||||
static def createTxtFile(ret, fileName, charset, isAppend){
|
||||
File file = new File(fileName);
|
||||
if (!file.getParentFile().exists()) {
|
||||
boolean result = file.getParentFile().mkdirs();
|
||||
if (!result) {
|
||||
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
|
||||
}
|
||||
}
|
||||
|
||||
OutputStream out = new FileOutputStream(fileName, isAppend);
|
||||
def sb = new StringBuffer();
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
def index = 0
|
||||
for(def row : ret){
|
||||
if(row){
|
||||
index++
|
||||
sb.append(row)
|
||||
sb.append("\n");
|
||||
}
|
||||
}
|
||||
if(sb.length()>0){
|
||||
out.write(sb.toString().getBytes(charset));
|
||||
sb.setLength(0)
|
||||
}
|
||||
out.close();
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -25,11 +25,11 @@ class DataMatch {
|
||||
static main(args) {
|
||||
def task = new DataMatch()
|
||||
// task.run1()
|
||||
task.run2()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// println task.findAddress("²ıÖݽֵÀÌÄÏã·")
|
||||
|
||||
// task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
task.run2MatchV2( "D:\\数据\\重庆农商行\\估值1-V2.xlsx")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
|
||||
+257
@@ -0,0 +1,257 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.Utils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class AnjukeAppCrawl {
|
||||
|
||||
public static final SHORT_DATE = Utils.getLastMonth();
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
public static def SOURCE = "安居客";
|
||||
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeAppCrawl();
|
||||
// task.run();
|
||||
// task.run2()
|
||||
|
||||
task.addThirdPrice()
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def url = "https://m.anjuke.com/sh/community/"
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def links = getRegion(url, driver)
|
||||
for(def link : links){
|
||||
def doc = getDocument(url, driver)
|
||||
def str = doc.toString().substring(doc.toString().indexOf("<script> window.__NUXT__"), doc.toString().indexOf("</script>"))
|
||||
if(str){
|
||||
str = str.substring(str.indexOf("} ("), str.indexOf("));"))
|
||||
def arrs = str.split(",")
|
||||
pritln arrs.size()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def getRegion(url, driver){
|
||||
//def doc = getDocument(url, driver)
|
||||
def header = [:]
|
||||
header.put("accept","application/json, text/plain, */*")
|
||||
header.put("accept-encoding","gzip, deflate, br")
|
||||
header.put("accept-language","zh-CN,zh;q=0.9")
|
||||
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
|
||||
header.put("referer","https://m.anjuke.com/fz/")
|
||||
header.put("sec-fetch-dest","empty")
|
||||
header.put("sec-fetch-mode","cors")
|
||||
header.put("sec-fetch-site","same-origin")
|
||||
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
|
||||
header.put("sw8-correlation","c291cmNl:YmVpZG91")
|
||||
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
|
||||
|
||||
def doc = ABuYunUtil.doGet(url, null, header, "utf-8")
|
||||
def links = []
|
||||
if(doc){
|
||||
def eles = JQueryUtils.select(".column2 a", doc)
|
||||
for(def e : eles){
|
||||
def href = e.attr("href")
|
||||
links << href
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
//上海:11,北京:14
|
||||
//"仓山":4071,
|
||||
// def map = ["晋安":4070,"鼓楼":4068,"闽侯":4073,"台江":4069,"福清":16378,"长乐":15880,"马尾":4072,"连江":16380,"平潭":16387,"罗源":16382,"永泰":16392,"闽清":16384]
|
||||
// def regionMap = ["浦东":7,"闵行":11, "松江":15,"宝山":13,"嘉定":14,"徐汇":5,"青浦":19,"静安":2,"普陀":10,"杨浦":9,"奉贤":17,"黄浦":4,"虹口":8,"长宁":6,"金山":18,"崇明":20]
|
||||
def regionMap = [ "朝阳":642,"海淀":601,"昌平":843,"丰台":753,"大兴":865,"通州":818,"房山":802,"顺义":837,"西城":704,"东城":690,"密云":856,"石景山":787,"怀柔":858,"门头沟":815,"延庆":860,"平谷":863,"北京周边":2678]
|
||||
def cityMap = ["北京":regionMap]
|
||||
def cityId = 14
|
||||
regionMap.each{region, area_id->
|
||||
for(def i=1;i<40;i++){
|
||||
def stop = false;
|
||||
def all_stop = false
|
||||
def error_count=0
|
||||
while(!stop){//渝北:2359 南岸:2368 沙坪坝:2380
|
||||
//城口
|
||||
def flagMap = getInfo(stop, i, area_id, cityId, "上海")
|
||||
stop = flagMap.stop
|
||||
all_stop = flagMap.all_stop
|
||||
error_count++
|
||||
if(error_count>=20){
|
||||
stop = true
|
||||
all_stop = true
|
||||
}
|
||||
}
|
||||
if(all_stop){
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getInfo(def stop, def i, def area_id, def cityId, def city){
|
||||
|
||||
def map = [:]
|
||||
def url = "https://m.anjuke.com/esf-ajax/community/list?city_id=${cityId}&page_size=100&page=${i}&area_id=${area_id}&is_tw=1"
|
||||
|
||||
def header = [:]
|
||||
header.put("accept","application/json, text/plain, */*")
|
||||
header.put("accept-encoding","gzip, deflate, br")
|
||||
header.put("accept-language","zh-CN,zh;q=0.9")
|
||||
header.put("cookie","aQQ_ajkguid=E9796514-38C0-44D2-8CA3-1C58C3928B29; sessid=765EEB9D-E7A2-4B2C-8151-CA9CD939AB02; id58=CpQB22J8sqMA9stuJfeAAg==; twe=2; 58tj_uuid=bf6eaff8-851c-471a-a914-853242878ddc; als=0; seo_source_type=1; _ga=GA1.2.215255179.1652344695; isp=true; ajk-appVersion=; obtain_by=2; fzq_js_anjuke_ershoufang_m=c2c12b2d1b221734fa20ef752cd6bdc0_1652509223606_27; fzq_h=ef0186eeaba082ecd2a47ede7c7e3e64_1652533612190_a34c026e9c1842a3b35bdd37f3d2aa0b_2015773878; init_refer=https%253A%252F%252Fm.anjuke.com%252Fcq%252Fcommunity%252F; new_uv=8; wmda_uuid=b59ce2efbba070ecfa8fe93d79f30dd4; wmda_new_uuid=1; wmda_session_id_6145577459763=1652535525740-dae51249-3379-d4e7; wmda_visited_projects=%3B6145577459763; new_session=0; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; xxzl_deviceid=q9XQbdA9hhkHJTj+RgVpwc7jrBzVJp9J1uTlp4hDtRiXtA615K4HS4R0pJ1dgrBa; ctid=20; xxzl_cid=4601bb59d47e4b42beb192b17946d1dd; fzq_js_anjuke_xiaoqu_m=c80eeca4a3301dbf27b3c79b9f18fa3b_1652536277223_27; xzuid=81992155-5990-4a2f-8842-32bcbfbb1117")
|
||||
header.put("referer","https://m.anjuke.com/sh/community/")
|
||||
header.put("sec-fetch-dest","empty")
|
||||
header.put("sec-fetch-mode","cors")
|
||||
header.put("sec-fetch-site","same-origin")
|
||||
header.put("sw8","1-ODA0ZDA3N2ZmMDA1NDgyYmExNTFlODhlZWIyNmFkMzk=-N2UwZDkxMGU2ZTdjNDdkZmJlZDBkMjUzMmNkODQ2NWU=-0-MTE3-aHR0cHM6Ly9tLmFuanVrZS5jb20=-L2VzZi1hamF4L2NvbW11bml0eS9saXN0-bS5hbmp1a2UuY29t-1")
|
||||
header.put("sw8-correlation","c291cmNl:YmVpZG91")
|
||||
header.put("user-agent","Mozilla/5.0 (iPhone; CPU iPhone OS 10_3_1 like Mac OS X) AppleWebKit/603.1.30 (KHTML, like Gecko) Version/10.0 Mobile/14E304 Safari/602.1")
|
||||
|
||||
def ret = ABuYunUtil.doGet(url, null, null, "utf-8")
|
||||
def all_stop = false;
|
||||
def error_count = 0
|
||||
if (ret) {
|
||||
JSONObject jsonObj = JSONObject.fromObject(ret);
|
||||
if(jsonObj && jsonObj.status=="ok" ){
|
||||
def data = jsonObj.data
|
||||
if(data && data.total.toInteger()>0){
|
||||
def communities = data.communities
|
||||
|
||||
println "【${i}】 "+communities.size() +" "+url
|
||||
if(communities.size()>0){
|
||||
save(city, communities)
|
||||
Thread.sleep(10000)
|
||||
stop = true;
|
||||
}else{
|
||||
stop = true;
|
||||
println url +" 返回0"
|
||||
all_stop = true;
|
||||
}
|
||||
|
||||
}else{
|
||||
stop = false;
|
||||
error_count ++
|
||||
println url +" 无数据"
|
||||
}
|
||||
|
||||
|
||||
}else{
|
||||
stop = false;
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}
|
||||
|
||||
map.stop = stop
|
||||
map.all_stop = all_stop;
|
||||
return map;
|
||||
}
|
||||
|
||||
def save(city, def ret){
|
||||
def index = 0
|
||||
for(def o : ret){
|
||||
index++
|
||||
def obj = findIsExist(o.jumpAction)
|
||||
if(!obj){
|
||||
def udate = new BasicDBObject()
|
||||
o.each{k,v->
|
||||
udate.put(k,v)
|
||||
}
|
||||
udate.put("city", city)
|
||||
udate.put("c_date", new Date())
|
||||
detail.save(udate)
|
||||
println index+" [保存]"+o.jumpAction
|
||||
}else{
|
||||
println index+" [已存在]"+o.jumpAction
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def findIsExist(def url){
|
||||
def q = new BasicDBObject()
|
||||
q.put("jumpAction",url)
|
||||
return detail.findOne(q);
|
||||
}
|
||||
|
||||
def addThirdPrice(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = detail.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
index++
|
||||
def city = d.base.cityName
|
||||
def region = d.base.areaName
|
||||
def name = d.base.name
|
||||
def obj = findByParams(city, region, name);
|
||||
if(!obj){
|
||||
def data = new BasicDBObject()
|
||||
data.put("district_id",d.base.id)
|
||||
data.put("city", city)
|
||||
data.put("region", region )
|
||||
data.put("name", name)
|
||||
data.put("address",d.base.address)
|
||||
data.put("avg_price", d.priceInfo.price)
|
||||
data.put("source", SOURCE)
|
||||
data.put("s_date", SHORT_DATE)
|
||||
data.put("c_date", d.c_date)
|
||||
thirdPartyColl.save(data)
|
||||
println index +" "+data
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findByParams(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("source", SOURCE)
|
||||
q.put("s_date", SHORT_DATE)
|
||||
return thirdPartyColl.findOne(q);
|
||||
}
|
||||
}
|
||||
+358
@@ -0,0 +1,358 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.WebElement
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.remote.DesiredCapabilities
|
||||
import org.openqa.selenium.remote.RemoteWebDriver
|
||||
|
||||
class AnjukeDetailCrawl {
|
||||
|
||||
DBCollection detailColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_detail")
|
||||
|
||||
DBCollection urlColl = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "detail_url")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeDetailCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://chongqing.anjuke.com/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
// DesiredCapabilities capabilities = DesiredCapabilities.chrome();
|
||||
// URL urlInstance = new URL("http://192.168.0.100:30025"); //指定URL
|
||||
// WebDriver driver = new RemoteWebDriver(urlInstance, capabilities);
|
||||
|
||||
driver.manage().deleteAllCookies()
|
||||
def regions = getRegions(driver, url)
|
||||
println "区域链接: "+regions.size()
|
||||
getAllRegions(driver, regions)
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getAllRegions(driver, regions){
|
||||
for(def url : regions){
|
||||
def doc = getDocument(url, driver)
|
||||
def total = getTotal(doc)
|
||||
if(total) {
|
||||
if (total > 1250) {
|
||||
def links = getMoreLinks(url, driver)
|
||||
println url +" "+links.size()
|
||||
saveDetailLinks(links)
|
||||
// parse(links,driver)
|
||||
} else {
|
||||
def links = getAllLinks(url, driver)
|
||||
println url +" "+links.size()
|
||||
saveDetailLinks(links)
|
||||
// parse(links,driver)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def saveDetailLinks(links){
|
||||
for(def url : links){
|
||||
def data = new BasicDBObject()
|
||||
data.put("city","重庆")
|
||||
data.put("url", url)
|
||||
data.put("source", "安居客")
|
||||
data.put("c_date", new Date())
|
||||
urlColl.save(data)
|
||||
}
|
||||
}
|
||||
|
||||
def getMoreLinks(url, driver){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def pricelinks = JQueryUtils.select(".filter-wrap:contains(均价) .line-item a", doc)
|
||||
def links = []
|
||||
if(pricelinks){
|
||||
for(def link : pricelinks){
|
||||
if(link.text()!="不限"){
|
||||
def urlLink = link.attr("href")
|
||||
getDocument(urlLink, driver)
|
||||
// driver.get(urlLink)
|
||||
links.addAll(getAllLinks(urlLink, driver))
|
||||
println urlLink +" "+links.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
def getAllLinks(url, driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def total = getTotal(doc)
|
||||
def page = getPage(total, 25)
|
||||
def pages = []
|
||||
for(def i=1;i<=page;i++){
|
||||
pages << url.substring(0, url.size()-1)+"-p"+i+"/"
|
||||
}
|
||||
for(def pageUrl : pages){
|
||||
doc = getDocument(pageUrl, driver)
|
||||
def details = JQueryUtils.select(".list-cell a.li-row", doc)
|
||||
for (def d : details) {
|
||||
def link = d.attr("href")
|
||||
if(!urls.contains(link)){
|
||||
urls << d.attr("href")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return urls
|
||||
}
|
||||
|
||||
def getPage(def total, def pageSize){
|
||||
if(total && total.toInteger()>0){
|
||||
def page = total.toInteger()/pageSize;
|
||||
if(page>(int)page){
|
||||
page = (int)page + 1
|
||||
return page
|
||||
}
|
||||
return (int)page;
|
||||
}
|
||||
return null
|
||||
}
|
||||
def getTotal(doc){
|
||||
|
||||
def totalText = JQueryUtils.selectText(".total-info", doc)
|
||||
if(totalText){
|
||||
def total = totalText.substring(totalText.indexOf("到")+1, totalText.indexOf("个")).trim()
|
||||
if(total && total.isNumber()){
|
||||
return total.toInteger()
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
def getRegions(driver, url){
|
||||
// def doc = getDocument(url, driver)
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def els = JQueryUtils.select("a:contains(小区)", doc)
|
||||
if(els){
|
||||
def href = els[0].attr("href")
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
// driver.findElement(By.xpath("//a[contains(text(), '小区')]")).click()
|
||||
// driver.findElement(By.linkText("小区")).click()
|
||||
doc = getDocument(href, driver)
|
||||
}
|
||||
|
||||
|
||||
// def loginIfm = JQueryUtils.select("#iframeLoginIfm", doc)
|
||||
// if(loginIfm){
|
||||
// def href = loginIfm[0].attr("src")
|
||||
// driver.get(href)
|
||||
// driver.findElementById("pwdTab").click()
|
||||
// WebElement elemUsername = driver.findElement(By.id("pwdUserNameIpt")); //获取用户名框
|
||||
// WebElement elemPassword = driver.findElement(By.id("pwdIpt")); //获取密码框
|
||||
// WebElement btn = driver.findElement(By.cssSelector("input[class='submit-btn']")); //获取点击登陆按钮。
|
||||
// elemUsername.sendKeys("17740818975");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
// btn.submit(); // 点击行为;
|
||||
//
|
||||
// def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
||||
// def urls = []
|
||||
// for(def e : regionEles){
|
||||
// urls << e.attr("href")
|
||||
// }
|
||||
// return urls;
|
||||
// }
|
||||
def regionEles = JQueryUtils.select(".region-parents li a", doc)
|
||||
def urls = []
|
||||
for(def e : regionEles){
|
||||
if(e.text()!="不限"){
|
||||
urls << e.attr("href")
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def getNontCheckPage(url, driver, doc){
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
if(doc){
|
||||
|
||||
}
|
||||
}
|
||||
return doc;
|
||||
}
|
||||
|
||||
def parse(def links, driver){
|
||||
def rets = []
|
||||
for(def link : links){
|
||||
def district = parsing(link, driver)
|
||||
if(district){
|
||||
// rets << district
|
||||
|
||||
// if (rets.size() % 100 == 0) {
|
||||
saveObj(district)
|
||||
println district.name+" "+district.url + " "+DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
// rets = []
|
||||
// }
|
||||
}
|
||||
}
|
||||
if (rets.size()> 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
saveObj(ret)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def saveObj(def ret){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = detailColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
detailColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
detailColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
}
|
||||
}
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc && !doc.toString().contains("访问验证-安居客") && !doc.toString().contains("502 Bad Gateway") && !doc.toString().contains("<title>登录</title>")){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
|
||||
def parsing(url, driver){
|
||||
|
||||
def district = [:]
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def headers = JQueryUtils.select(".crumbs a", doc)
|
||||
if(headers){
|
||||
if (headers && headers.size() >= 3) {
|
||||
district.city = headers[1].text().replace("小区", "")
|
||||
district.region = headers[2].text().replace("小区", "")
|
||||
district.name = headers[headers.size() - 1].text().replace("二手房", "")
|
||||
}
|
||||
|
||||
|
||||
district.address = JQueryUtils.selectText("div.column-1:contains(小区地址) .value", doc)
|
||||
|
||||
def priceStr = JQueryUtils.selectText(".average", doc)
|
||||
|
||||
if (priceStr && priceStr.isNumber()) {
|
||||
district.avg_price = priceStr.toInteger()
|
||||
}
|
||||
|
||||
def infoDiv = JQueryUtils.select(".info-list", doc)
|
||||
district.estate_type = getValue(infoDiv.select(".column-2:contains(物业类型) .value").text())
|
||||
district.quanshu_type = getValue(infoDiv.select(".column-2:contains(权属类别) .value").text())
|
||||
district.finish_date = getValue(infoDiv.select(".column-2:contains(竣工时间) .value").text())
|
||||
district.property_life = getValue(infoDiv.select(".column-2:contains(产权年限) .value").text())
|
||||
def houseCountStr = infoDiv.select(".column-2:contains(总户数) .value").text()
|
||||
if (houseCountStr) {
|
||||
houseCountStr = houseCountStr.replace("户", "")
|
||||
if (houseCountStr) {
|
||||
district.house_count = houseCountStr.toInteger()
|
||||
}
|
||||
}
|
||||
district.build_area = getValue(infoDiv.select(".column-2:contains(总建面积) .value").text())
|
||||
district.volumetric_rate = getValue(infoDiv.select(".column-2:contains(容积率) .value").text())
|
||||
district.greening_rate = getValue(infoDiv.select(".column-2:contains(绿化率) .value").text())
|
||||
|
||||
district.build_type = getValue(infoDiv.select(".column-2:contains(建筑类型) .value").text())
|
||||
district.business = getValue(infoDiv.select(".column-2:contains(所属商圈) .value").text())
|
||||
|
||||
def carSiteStr = getValue(infoDiv.select(".column-2:contains(停车位) .value").text())
|
||||
if(carSiteStr){
|
||||
if(carSiteStr.contains("(")){
|
||||
carSiteStr = carSiteStr.substring(0, carSiteStr.indexOf("("))
|
||||
}
|
||||
if(carSiteStr && carSiteStr.isNumber()){
|
||||
district.car_site = carSiteStr
|
||||
}
|
||||
}
|
||||
district.car_site_str = infoDiv.select(".column-2:contains(停车位) .value").text()
|
||||
district.estate_charge = getValue(infoDiv.select(".column-2:contains(物业费) .value").text())
|
||||
|
||||
district.stop_charge = getValue(infoDiv.select(".column-1:contains(停车费) .value").text())
|
||||
district.car_site_charge = getValue(infoDiv.select(".column-1:contains(车位管理费) .value").text())
|
||||
|
||||
district.estate_company = getValue(infoDiv.select(".column-1:contains(物业公司) .value").text())
|
||||
district.developer = getValue(infoDiv.select(".column-1:contains(开发商) .value").text())
|
||||
district.url = url
|
||||
district.c_date = new Date()
|
||||
district.m_date = new Date()
|
||||
district.crawl_type = "chrome"
|
||||
district.root = "anjuke.com"
|
||||
stop = true;
|
||||
}else{
|
||||
Thread.sleep(3000)
|
||||
count++
|
||||
}
|
||||
if(count>10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
return district
|
||||
}
|
||||
|
||||
String getValue(String s){
|
||||
if(s && s.contains("暂无")){
|
||||
return null
|
||||
}
|
||||
return s
|
||||
}
|
||||
}
|
||||
+89
@@ -0,0 +1,89 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.temp.shenjian.util.SeaweedUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
class AnjukeDistrictMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeDistrictMatch();
|
||||
task.run();
|
||||
// task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
def regionPointsMap = findRegionRange("重庆")
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
|
||||
}
|
||||
def run(){
|
||||
|
||||
|
||||
|
||||
def filePath = "D:\\数据\\重庆农商行\\安居客\\重庆小区.xlsx"
|
||||
ExcelUtils.multiThreadingModify(filePath,{row->
|
||||
if(row['A'] && !row['J']){
|
||||
def city = row['B']
|
||||
def region = row['C']
|
||||
def name = row['D']
|
||||
def address = row['E']
|
||||
// def flag = "别名匹配"
|
||||
|
||||
|
||||
// def dist = matchDist(city, null, name)
|
||||
// if(!row['J'] && name.contains("(")){
|
||||
// name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||
// flag = "去括号匹配"
|
||||
// dist = matchDist(city, region, name)
|
||||
// }
|
||||
// if(!dist){
|
||||
// dist = CommonDatautils.findSeaweedByMappingAlias(city, region, name,"district")
|
||||
// flag = "映射别名匹配"
|
||||
// }
|
||||
// if(!dist) {
|
||||
// def rets = CommonDatautils.findDistrictByAlias(city, region, name, "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }
|
||||
|
||||
//
|
||||
// if(!dist){
|
||||
// def dists = CommonDatautils.findDistrictByAddress(city, region, address,"district")
|
||||
// flag = "地址匹配"
|
||||
// if(dists.size()>1){
|
||||
// println address +" 地址相同:"+dists.size()
|
||||
// }else{
|
||||
// if(dists.size()==1){
|
||||
// dist = dists[0]
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(dist){
|
||||
// row['J'] = flag
|
||||
// row['K'] = dist._id.toString()
|
||||
// row['L'] = dist.region
|
||||
// row['M'] = dist.name
|
||||
// row['N'] = dist.address
|
||||
// }
|
||||
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false,region,name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
// row['J'] = "匹配接口"
|
||||
// row['K'] = jsonObject.get("id")
|
||||
// row['L'] = jsonObject.get("district")
|
||||
// row['M'] = jsonObject.get("name")
|
||||
// row['N'] = jsonObject.get("address")
|
||||
row['M'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchDist(city, region, name){
|
||||
def dist = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
return dist;
|
||||
}
|
||||
}
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.alibaba.fastjson.JSON
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.temp.shenjian.util.MessageUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class AnjukeHouseCrawl {
|
||||
def targetUrl = "http://192.168.0.206:9200/amap_poi/_doc"
|
||||
static main(args) {
|
||||
def task = new AnjukeHouseCrawl();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
for(def i=24;i<50;i++){
|
||||
def url = "https://chengde.anjuke.com/sale/shuangqiao/p${i}/?from=navigation"
|
||||
def doc = getDocument(url, driver)
|
||||
def eles = JQueryUtils.select(".list-main .list-left .list .property a", doc)
|
||||
println "页数:"+i+" 记录数:"+eles.size()
|
||||
if(eles){
|
||||
parser(eles, driver)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def parser(eles, driver){
|
||||
def index = 0
|
||||
for(def e : eles){
|
||||
index ++
|
||||
def href = e.attr("href")
|
||||
def doc = getDocument(href, driver)
|
||||
// parserHtml(doc)
|
||||
def newUrl = href.substring(0, href.indexOf("?"))
|
||||
sendMq(newUrl, doc)
|
||||
|
||||
println "[ "+index +"] "+ newUrl
|
||||
}
|
||||
}
|
||||
|
||||
def sendMq(href, doc){
|
||||
JSON json = new JSONObject()
|
||||
json.put("project", "sell_anjuke")
|
||||
json.put("url", href)
|
||||
|
||||
JSON result = new JSONObject()
|
||||
result.put("content", doc.toString())
|
||||
json.put("result", result)
|
||||
|
||||
byte[] packedMsg = MessageUtils.pack(json)
|
||||
CommonUtils.sendMessage("result2parse", packedMsg)
|
||||
}
|
||||
|
||||
def parserHtml(doc) {
|
||||
String city = null
|
||||
def result = [:]
|
||||
result.siteCode = "安居客"
|
||||
if (doc) {
|
||||
result.title = JQueryUtils.selectText([".banner-title h1"], doc)
|
||||
def heads = JQueryUtils.select(".crumbs a", doc)
|
||||
|
||||
if (heads && heads.size() >= 5) {
|
||||
result.city = heads[0].text().replace("房产网", "")
|
||||
def orgRegion = heads[2].text().replace("二手房", "")
|
||||
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
|
||||
result.block = heads[3].text().replace("二手房", "")
|
||||
result.district_name = heads[4].text()
|
||||
} else if (heads && heads.size() >= 3) {
|
||||
result.city = heads[0].text().replace("房产网", "")
|
||||
result.district_name = heads[2].text()
|
||||
def eles = JQueryUtils.select("li:contains(所在位置:) a", doc)
|
||||
if (eles && eles.size() > 0) {
|
||||
def orgRegion = eles[0].text()
|
||||
result.region = RegionData.formatRegion(result.city, orgRegion.trim())
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.district_name) {
|
||||
println " [未解析出(城市-区域-名称)信息]"
|
||||
}
|
||||
println result
|
||||
|
||||
|
||||
def priceStr = JQueryUtils.selectText(".maininfo-price-num", doc)
|
||||
if (priceStr) {
|
||||
if (priceStr.isNumber()) {
|
||||
result.total_price = priceStr.toDouble() * 10000
|
||||
}
|
||||
}
|
||||
def roomInfoStr = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-strong", doc)
|
||||
if (roomInfoStr) {
|
||||
Integer room = DataCrawlUtils.parseRoom(roomInfoStr)
|
||||
Integer hall = DataCrawlUtils.parseHall(roomInfoStr)
|
||||
Integer toilet = DataCrawlUtils.parseToilet(roomInfoStr)
|
||||
|
||||
if (room) {
|
||||
result.room = room
|
||||
}
|
||||
if (hall) {
|
||||
result.hall = hall
|
||||
}
|
||||
if (toilet) {
|
||||
result.toilet = toilet
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-strong", doc).replace("㎡", "")
|
||||
if (areaStr) {
|
||||
result.area = areaStr.toDouble()
|
||||
}
|
||||
def directionStr = JQueryUtils.selectText(".maininfo-model-item-3 i", doc)
|
||||
result.direction = DataCrawlUtils.parseDirection(directionStr)
|
||||
|
||||
def buildDateStr = JQueryUtils.selectText(".maininfo-model-item-3 .maininfo-model-weak", doc)
|
||||
if (buildDateStr && buildDateStr.contains("/")) {
|
||||
def arrs = buildDateStr.split("\\/")
|
||||
if (arrs && arrs.size() == 2) {
|
||||
result.build_date = arrs[0]
|
||||
result.estate_type = arrs[1]
|
||||
}
|
||||
}
|
||||
|
||||
result.floor_info = JQueryUtils.selectText(".maininfo-model-item-1 .maininfo-model-weak", doc)
|
||||
if (result.floor_info && result.floor_info.contains("共")) {
|
||||
def height = result.floor_info
|
||||
if (result.floor_info.contains("(")) {
|
||||
height = height.substring(result.floor_info.indexOf("(") + 1, result.floor_info.indexOf(")"))
|
||||
}
|
||||
height = height.replace("共", "").replace("层", "")
|
||||
if (height) {
|
||||
result.height = height.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (result.area && result.total_price) {
|
||||
result.avg_price = (result.total_price / result.area).toInteger()
|
||||
}
|
||||
|
||||
def fitmitStr = JQueryUtils.selectText(".maininfo-model-item-2 .maininfo-model-weak", doc)
|
||||
result.fitment = DataCrawlUtils.parseFitment(fitmitStr)
|
||||
|
||||
|
||||
def dateStr = JQueryUtils.selectText(["td:contains(发布时间) .houseInfo-main-item-name"], doc)
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
Thread.sleep(5000)
|
||||
def els = JQueryUtils.select(".list-main .list-left .list", doc)
|
||||
if(!els){
|
||||
els = JQueryUtils.select(".props-main", doc)
|
||||
}
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
}
|
||||
+631
@@ -0,0 +1,631 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class AnjukeToDB {
|
||||
|
||||
DBCollection detail = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "app_anjuke_detail")
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeToDB();
|
||||
task.run();
|
||||
// task.addNoMatch()
|
||||
// task.updateEstateType()
|
||||
// task.gaodeRangeData()
|
||||
}
|
||||
|
||||
def updateEstateType(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客重庆小区-曹玉彦.xlsx",0,1)
|
||||
for(def row : datas){
|
||||
def anjukeId = row['A']
|
||||
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||
if(obj) {
|
||||
def type = getEstatType(obj.extend.type)
|
||||
def commId = row['L']
|
||||
if(!commId){
|
||||
commId = row['G']
|
||||
}
|
||||
if(type){
|
||||
if(type=="别墅"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("estate_type2", type)
|
||||
udata.put("m_date", new Date())
|
||||
println commId
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId.trim())), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}else{
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("status", 1)
|
||||
// udata.put("name_cat",11)
|
||||
// udata.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(commId)), new BasicDBObject('$set', udata))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def gaodeRangeData(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\徐鹤高德地图.xlsx",0,1)
|
||||
def index = 0
|
||||
for(def d : datas){
|
||||
index++
|
||||
println index
|
||||
def personName = d['AB']
|
||||
if(personName){
|
||||
def commid = d['J']
|
||||
if(commid){
|
||||
def obj = CommonDatautils.findDistrict("重庆",d['C'],personName,"district")
|
||||
if(obj){
|
||||
updateDist(d, obj._id.toString())
|
||||
}
|
||||
}else{
|
||||
saveGaode(d)
|
||||
}
|
||||
}else{
|
||||
def commid = d['J']
|
||||
updateDist(d, commid)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def saveGaode(d){
|
||||
def oldName = d['E']
|
||||
def distName = nameFileter(d['C'], oldName)
|
||||
def udata = new BasicDBObject()
|
||||
def alias = []
|
||||
|
||||
def aliasStr = d['F']
|
||||
if(aliasStr){
|
||||
def arrs = aliasStr.split("/")
|
||||
for(def s : arrs){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
if(alias.size()>0){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
|
||||
def address = AddressUtil.formatAddress("重庆", d['G'])
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
if(d['N']){
|
||||
udata.put("virescence_percent", d['N'])
|
||||
}
|
||||
|
||||
if(d['O']){
|
||||
udata.put("estate_charge", d['O'])
|
||||
}
|
||||
if(d['Q']){
|
||||
udata.put("cubage_percent", d['Q'])
|
||||
}
|
||||
if(d['R']){
|
||||
udata.put("developer", d['R'])
|
||||
}
|
||||
|
||||
if( d['S']){
|
||||
udata.put("build_area", d['S'])
|
||||
}
|
||||
|
||||
if(d['T']){
|
||||
def t = d['T'].substring(0, 3)
|
||||
udata.put("build_date", t)
|
||||
}
|
||||
|
||||
if(d['V'] && d['V'].isNumber()){
|
||||
udata.put("car_site_count", d['V'].toInteger())
|
||||
}
|
||||
|
||||
if(d['Y']){
|
||||
def build_type = getBuildType2(d['Y'])
|
||||
if(build_type){
|
||||
udata.put("build_type", build_type)
|
||||
}
|
||||
}
|
||||
|
||||
if(d['Z']){
|
||||
udata.put("property_life", d['Z'])
|
||||
}
|
||||
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(d['H']){
|
||||
lat = d['H']
|
||||
lng = d['I']
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
|
||||
Long dLat2 = lat2 * 10000000000L
|
||||
Long dLng2 = lng2 * 10000000000L
|
||||
udata.put("lat2", dLat2)
|
||||
udata.put("lng2", dLng2)
|
||||
Long dlat = lat.toDouble() * 10000000000L
|
||||
Long dlng = lng.toDouble() * 10000000000L
|
||||
udata.put("lat", dlat)
|
||||
udata.put("lng", dlng)
|
||||
}
|
||||
def type = "普通住宅"
|
||||
if(distName.contains("别墅")) {
|
||||
type = "别墅"
|
||||
}else if(distName.contains("商住")){
|
||||
type = "商住"
|
||||
}
|
||||
|
||||
if(udata){
|
||||
udata.put("name_cat",0)
|
||||
udata.put("status",0)
|
||||
udata.put("cat","district")
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("city", "重庆")
|
||||
udata.put("region",d['C'])
|
||||
udata.put("name", distName)
|
||||
udata.put("estate_type2",type)
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
udata.put("mark_map",markMap)
|
||||
DbUtils.seaweed.save(udata);
|
||||
|
||||
def stop = false
|
||||
while (!stop){
|
||||
if(udata._id.toString()){
|
||||
addPrice(udata)
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def updateDist(d, commid){
|
||||
|
||||
if(commid){
|
||||
def dist = CommonDatautils.queryDistrictById(commid)
|
||||
if(dist){
|
||||
def oldName = d['E']
|
||||
def distName = nameFileter(d['C'], oldName)
|
||||
def udata = new BasicDBObject()
|
||||
if(!distName.contains("号")){
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(distName) && dist.name!=distName){
|
||||
alias << distName
|
||||
}
|
||||
def aliasStr = d['F']
|
||||
if(aliasStr){
|
||||
def arrs = aliasStr.split("/")
|
||||
for(def s : arrs){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
if(alias.size()>0 && alias!=dist.alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
}
|
||||
|
||||
if(dist.address && !dist.address.contains("号")){
|
||||
def address = AddressUtil.formatAddress("重庆", d['G'])
|
||||
if(address){
|
||||
udata.put("address", address)
|
||||
}
|
||||
}
|
||||
|
||||
if(!dist.virescence_percent && d['N']){
|
||||
udata.put("virescence_percent", d['N'])
|
||||
}
|
||||
|
||||
if(!dist.estate_charge && d['O']){
|
||||
udata.put("estate_charge", d['O'])
|
||||
}
|
||||
if(!dist.cubage_percent && d['Q']){
|
||||
udata.put("cubage_percent", d['Q'])
|
||||
}
|
||||
if(!dist.developer && d['R']){
|
||||
udata.put("developer", d['R'])
|
||||
}
|
||||
|
||||
if(!dist.build_area && d['S']){
|
||||
udata.put("build_area", d['S'])
|
||||
}
|
||||
|
||||
if(!dist.build_date && d['T']){
|
||||
if(d['T'].size()>=4){
|
||||
def t = d['T'].substring(0, 4)
|
||||
udata.put("build_date", t)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!dist.car_site_count && d['V'] && d['V'].isNumber()){
|
||||
udata.put("car_site_count", d['V'].toInteger())
|
||||
}
|
||||
|
||||
if(!dist.build_type && d['Y']){
|
||||
def build_type = getBuildType2(d['Y'])
|
||||
if(build_type){
|
||||
udata.put("build_type", build_type)
|
||||
}
|
||||
}
|
||||
|
||||
if(!dist.property_life && d['Z']){
|
||||
udata.put("property_life", d['Z'])
|
||||
}
|
||||
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(d['H']){
|
||||
lat = d['H']
|
||||
lng = d['I']
|
||||
|
||||
def slat = dist.lat/10000000000.0
|
||||
def slng = dist.lng/10000000000.0
|
||||
|
||||
def distance = GeoUtils.calculateDistance(lng.toDouble(), lat.toDouble(), slng, slat);
|
||||
if(distance>1000){
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng.toDouble(), lat.toDouble(), 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
|
||||
Long dLat2 = lat2 * 10000000000L
|
||||
Long dLng2 = lng2 * 10000000000L
|
||||
udata.put("lat2", dLat2)
|
||||
udata.put("lng2", dLng2)
|
||||
Long dlat = lat.toDouble() * 10000000000L
|
||||
Long dlng = lng.toDouble() * 10000000000L
|
||||
udata.put("lat", dlat)
|
||||
udata.put("lng", dlng)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改] ${dist.name}"+ udata
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addNoMatch(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\安居客未匹配.xlsx",0,1)
|
||||
for(def d : datas){
|
||||
add(d)
|
||||
}
|
||||
}
|
||||
def run(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\重庆农商行\\安居客\\非人工处理.xlsx",0,1)
|
||||
for(def data : datas){
|
||||
// if(data['K']=="镕杰"){
|
||||
def flag = data['M']
|
||||
if(flag && flag=="新增"){
|
||||
// add(data)
|
||||
}else{
|
||||
update(data)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def add(def row){
|
||||
def anjukeId = row['A']
|
||||
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||
if(obj){
|
||||
def type = getEstatType(obj.extend.type)
|
||||
if(type){
|
||||
def city = obj.base.cityName
|
||||
def region = obj.base.areaName
|
||||
def lat2 = null
|
||||
def lng2 = null
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(obj.base.lat){
|
||||
lat2 = obj.base.lat
|
||||
lng2 = obj.base.lng
|
||||
|
||||
JSONObject convertObj = GaoDeApiUtils.convert(lng2+","+lat2, "baidu")
|
||||
if(convertObj.getInt("status") == 1) {
|
||||
String lo = convertObj.get("locations")
|
||||
lng = Double.parseDouble(lo.split(",")[0])
|
||||
lat = Double.parseDouble(lo.split(",")[1])
|
||||
}
|
||||
}
|
||||
|
||||
def distName = nameFileter(obj.base.areaName, obj.base.name)
|
||||
|
||||
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
|
||||
|
||||
def address = AddressUtil.formatAddress(obj.base.city, obj.base.address)
|
||||
|
||||
def dist = CommonDatautils.findDistrict(city, region, distName, "district")
|
||||
if(!dist){
|
||||
def data = new BasicDBObject();
|
||||
data.put("name_cat",0)
|
||||
data.put("status",0)
|
||||
data.put("cat","district")
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
data.put("city", city)
|
||||
data.put("region",region)
|
||||
data.put("name", distName)
|
||||
data.put("estate_type2",type)
|
||||
if(address){
|
||||
data.put("address", obj.base.address)
|
||||
}
|
||||
|
||||
if(lat){
|
||||
Long alat = lat*10000000000L
|
||||
Long alng = lng*10000000000L
|
||||
data.put("lat", alat)
|
||||
data.put("lng", alng)
|
||||
Long alat2 = lat2.toDouble()*10000000000L
|
||||
Long alng2 = lng2.toDouble()*10000000000L
|
||||
data.put("lat2",alat2)
|
||||
data.put("lng2",alng2)
|
||||
}
|
||||
data.put("finish_building_date",obj.extend.completionTime)
|
||||
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber()){
|
||||
data.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
|
||||
}
|
||||
data.put("build_area", obj.extend.totalArea)
|
||||
|
||||
data.put("virescence_percent", obj.extend.landscapingRatio)
|
||||
data.put("build_type", buildTypeStr)
|
||||
data.put("business_quarter",obj.base.tradingAreaName)
|
||||
|
||||
|
||||
def markMap = [:]
|
||||
markMap.put("data_source", "机器入库")
|
||||
data.put("mark_map",markMap)
|
||||
DbUtils.seaweed.save(data);
|
||||
|
||||
def stop = false
|
||||
while (!stop){
|
||||
if(data._id.toString()){
|
||||
addPrice(data)
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}else{
|
||||
println "【物业类型问题】 ${obj._id.toString()} "+obj.extend.type
|
||||
}
|
||||
}else{
|
||||
println anjukeId+" 没找到"
|
||||
}
|
||||
|
||||
}
|
||||
def update(def row){
|
||||
def commId = row['L']
|
||||
if(!commId){
|
||||
commId = row['G']
|
||||
}
|
||||
|
||||
def dist = CommonDatautils.queryDistrictById(commId)
|
||||
def anjukeId = row['A']
|
||||
def obj = findAnjukeDistrictDetail(anjukeId)
|
||||
if(obj && dist){
|
||||
def udata = new BasicDBObject()
|
||||
|
||||
def address = AddressUtil.formatAddress(obj.base.cityName, obj.base.address)
|
||||
if(address){
|
||||
if(!dist.address){
|
||||
if(obj.base.address){
|
||||
udata.put("address",obj.base.address)
|
||||
}
|
||||
}else{
|
||||
if(dist.address.contains("号")){
|
||||
if(!dist.address.contains(obj.base.address) && !obj.base.address.contains(dist.address)){
|
||||
def addr_alias = dist.addr_alias
|
||||
if(!addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
if(!addr_alias.contains(obj.base.address) && obj.base.address.contains("号")){
|
||||
addr_alias << obj.base.address
|
||||
udata.put("addr_alias", addr_alias)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
udata.put("address",obj.base.address)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def distName = nameFileter(obj.base.areaName, obj.base.name)
|
||||
if(!distName.contains("号")){
|
||||
def alias = dist.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(alias.contains(distName)){
|
||||
alias << distName
|
||||
}
|
||||
if(alias.size()>0 && alias!=dist.alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
}
|
||||
|
||||
if(!dist.build_type){
|
||||
def buildTypeStr = getBuildType(obj.base.buildTypeStr)
|
||||
if(buildTypeStr){
|
||||
udata.put("build_type",buildTypeStr)
|
||||
}
|
||||
|
||||
}
|
||||
if(!dist.household_count){
|
||||
if(obj.extend.totalHouseHoldNum && obj.extend.totalHouseHoldNum.isNumber() && obj.extend.totalHouseHoldNum.toInteger()>0){
|
||||
udata.put("household_count",obj.extend.totalHouseHoldNum.toInteger())
|
||||
}
|
||||
}
|
||||
if(!dist.finish_building_date){
|
||||
if(obj.extend.completionTime){
|
||||
udata.put("finish_building_date",obj.extend.completionTime)
|
||||
}
|
||||
}
|
||||
|
||||
if(!dist.virescence_percent){
|
||||
if(obj.extend.landscapingRatio){
|
||||
udata.put("virescence_percent", obj.extend.landscapingRatio)
|
||||
}
|
||||
}
|
||||
if(!dist.business_quarter){
|
||||
if(obj.base.tradingAreaName){
|
||||
udata.put("business_quarter",obj.base.tradingAreaName)
|
||||
}
|
||||
}
|
||||
def lat2 = null
|
||||
def lng2 = null
|
||||
if(dist.lat2){
|
||||
lat2 = dist.lat2/10000000000.0
|
||||
lng2 = dist.lng2/10000000000.0
|
||||
}
|
||||
if(dist.lat && !lat2){
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(dist.lng/10000000000.0, dist.lat/10000000000.0, 3)
|
||||
lng2 = baidu.x
|
||||
lat2 = baidu.y
|
||||
|
||||
Long dLat2 = lat2 * 10000000000L
|
||||
Long dLng2 = lng2 * 10000000000L
|
||||
udata.put("lat2", dLat2)
|
||||
udata.put("lng2", dLng2)
|
||||
}
|
||||
|
||||
if(obj.base.lat) {
|
||||
def alat2 = obj.base.lat.toDouble()
|
||||
def alng2 = obj.base.lng.toDouble()
|
||||
def distance = GeoUtils.calculateDistance(lng2, lat2, alng2, alat2);
|
||||
if(distance>1000){
|
||||
println "........${obj.base.name}..............${dist.name}..........距离相差${distance}......................"
|
||||
|
||||
def lng = null
|
||||
def lat = null
|
||||
JSONObject convertObj = GaoDeApiUtils.convert(obj.base.lng+","+obj.base.lat, "baidu")
|
||||
if(convertObj.getInt("status") == 1) {
|
||||
String lo = convertObj.get("locations")
|
||||
lng = Double.parseDouble(lo.split(",")[0])
|
||||
lat = Double.parseDouble(lo.split(",")[1])
|
||||
}
|
||||
|
||||
Long blat2 = alat2 *10000000000L
|
||||
Long blng2 = alng2 *10000000000L
|
||||
Long gLat2 = lat *10000000000L
|
||||
Long gLng2 = lng *10000000000L
|
||||
|
||||
udata.put("lat2", blat2)
|
||||
udata.put("lng2", blng2)
|
||||
udata.put("lat", gLat2)
|
||||
udata.put("lng", gLng2)
|
||||
}
|
||||
}
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改] ${dist.name}"+ udata
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def findAnjukeDistrictDetail(anjukeId){
|
||||
def q = new BasicDBObject()
|
||||
try{
|
||||
q.put("_id",new ObjectId(anjukeId))
|
||||
return detail.findOne(q)
|
||||
}catch(Exception e){
|
||||
print "[查询出错]anjukeId:"+anjukeId
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def nameFileter(def region, def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
name = name.replace("(${region})","").replace("(","").replace(")","")
|
||||
return name;
|
||||
}
|
||||
|
||||
def getEstatType(type){
|
||||
if(type){
|
||||
if(type=="公寓住宅"){
|
||||
type = "普通住宅"
|
||||
}else if(type.contains("商住")){
|
||||
type = "商住"
|
||||
}else if(type.contains("别墅")){
|
||||
type = "别墅"
|
||||
}else{
|
||||
type = null
|
||||
}
|
||||
}
|
||||
return type
|
||||
}
|
||||
|
||||
def getBuildType(def buildType){
|
||||
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
|
||||
def list = []
|
||||
if(buildType){
|
||||
def arrs = buildType.split("\\|")
|
||||
for(def s : arrs){
|
||||
if(types.contains(s)){
|
||||
list << s
|
||||
}
|
||||
}
|
||||
if(list){
|
||||
return list.join(",")
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getBuildType2(def buildType){
|
||||
def types = ["多层","小高层","高层","超高层","花园洋房","老洋房","四合院","独栋","双拼","联排","叠加","老式别墅洋房"]
|
||||
def list = []
|
||||
if(buildType){
|
||||
def arrs = buildType.split(",| ")
|
||||
for(def s : arrs){
|
||||
if(types.contains(s)){
|
||||
list << s
|
||||
}
|
||||
}
|
||||
if(list){
|
||||
return list.join(",")
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def addPrice(dist){
|
||||
def startDate = DateUtils.parseFromTightYYYYMMDD("2022-04-01")
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2022-05-25")
|
||||
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
|
||||
|
||||
for(def weekend : weekends){
|
||||
CommonUtils.recalculationAverage(dist.city, dist.region, dist.name, dist.estate_type2, weekend,false,0)
|
||||
}
|
||||
}
|
||||
}
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
class CommonCrawl {
|
||||
static def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
}
|
||||
+250
@@ -0,0 +1,250 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataCrawlUtils {
|
||||
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
|
||||
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
|
||||
static {
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
|
||||
})
|
||||
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
|
||||
})
|
||||
|
||||
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
|
||||
str = str.replaceAll("室|房", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
|
||||
str = str.replace("厅", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
|
||||
str = str.replace("卫", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
|
||||
str = str.replace("万", '').trim()
|
||||
return (str.toDouble() * 10000).toInteger()
|
||||
})
|
||||
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
|
||||
str = str.replace("元/㎡", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m?"), { String str ->
|
||||
str = str.replace("元/m?", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
|
||||
str = str.replace("元/平", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
|
||||
str = str.replace("元", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
|
||||
str = str.replace("平米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
|
||||
str = str.replace("平方米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
|
||||
str = str.replace("平", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
|
||||
str = str.replace("㎡", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m?"), { String str ->
|
||||
str = str.replace("m?", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
|
||||
str = str.replace("共", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
|
||||
str = str.replace("总", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n2
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n1
|
||||
}
|
||||
return n1
|
||||
})
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
|
||||
str = str.replace("第", "").replaceAll("层|楼", "")
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析室数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseRoom(String content) {
|
||||
return parse(content, ROOM_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHall(String content) {
|
||||
return parse(content, HALL_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseToilet(String content) {
|
||||
return parse(content, TOILET_RESOLVERS)
|
||||
}
|
||||
|
||||
static Object parse(String content, Map<Pattern, Closure> resolvers) {
|
||||
if (content != null) {
|
||||
content = content.replaceAll("\\s", "")
|
||||
}
|
||||
for (def resolver : resolvers.entrySet()) {
|
||||
String match = match(content, resolver.getKey())
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
|
||||
return resolver.getValue()(match)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 成交数据解析工具类
|
||||
*/
|
||||
class DealPriceUtils {
|
||||
|
||||
static final Map<Pattern, Closure> DEAL_DATE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> ROOM_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HALL_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOILET_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> TOTAL_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> UNIT_PRICE_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> AREA_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> HEIGHT_RESOLVERS = [:]
|
||||
static final Map<Pattern, Closure> FLOOR_RESOLVERS = [:]
|
||||
static final List<String> DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
|
||||
static {
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}-\\d{2}-\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{2}/\\d{2}/\\d{2}"), { String str ->
|
||||
str = "20" + str
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy/MM/dd")
|
||||
})
|
||||
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}\\.\\d{2}\\.\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy.MM.dd")
|
||||
})
|
||||
DEAL_DATE_RESOLVERS.put(Pattern.compile("\\d{4}-\\d{2}"), { String str ->
|
||||
return DateUtils.parseFromTightByPattern(str, "yyyy-MM")
|
||||
})
|
||||
|
||||
ROOM_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两](室|房)"), { String str ->
|
||||
str = str.replaceAll("室|房", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HALL_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]厅"), { String str ->
|
||||
str = str.replace("厅", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOILET_RESOLVERS.put(Pattern.compile("[\\d+|一|二|三|四|五|两]卫"), { String str ->
|
||||
str = str.replace("卫", '').trim()
|
||||
return str.toInteger()
|
||||
})
|
||||
TOTAL_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?万"), { String str ->
|
||||
str = str.replace("万", '').trim()
|
||||
return (str.toDouble() * 10000).toInteger()
|
||||
})
|
||||
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/㎡"), { String str ->
|
||||
str = str.replace("元/㎡", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/m²"), { String str ->
|
||||
str = str.replace("元/m²", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元/平"), { String str ->
|
||||
str = str.replace("元/平", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
UNIT_PRICE_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?元"), { String str ->
|
||||
str = str.replace("元", '').trim()
|
||||
return str.toDouble().toInteger()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平米"), { String str ->
|
||||
str = str.replace("平米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平方米"), { String str ->
|
||||
str = str.replace("平方米", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?平"), { String str ->
|
||||
str = str.replace("平", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?㎡"), { String str ->
|
||||
str = str.replace("㎡", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
AREA_RESOLVERS.put(Pattern.compile("\\d+(\\.\\d+)?m²"), { String str ->
|
||||
str = str.replace("m²", '').trim()
|
||||
return str.toDouble()
|
||||
})
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("共\\d+层"), { String str ->
|
||||
str = str.replace("共", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("总\\d+层"), { String str ->
|
||||
str = str.replace("总", '').replace('层', '')
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
HEIGHT_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n2
|
||||
}
|
||||
return null
|
||||
})
|
||||
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("(?<=\\D)(\\d{1,2}/[1-6]?\\d)(?=\\D)"), { String str ->
|
||||
def split = str.split("/")
|
||||
def n1 = split[0].toInteger()
|
||||
def n2 = split[1].toInteger()
|
||||
if (n1 <= n2) {
|
||||
return n1
|
||||
}
|
||||
return n1
|
||||
})
|
||||
|
||||
FLOOR_RESOLVERS.put(Pattern.compile("第\\d+(层|楼)"), { String str ->
|
||||
str = str.replace("第", "").replaceAll("层|楼", "")
|
||||
return str.toInteger()
|
||||
})
|
||||
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
|
||||
}
|
||||
/**
|
||||
* 解析成交日期
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Date parseDealDate(String content) {
|
||||
return parse(content, DEAL_DATE_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析室数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseRoom(String content) {
|
||||
return parse(content, ROOM_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHall(String content) {
|
||||
return parse(content, HALL_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析厅数
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseToilet(String content) {
|
||||
return parse(content, TOILET_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 解析总价
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseTotalPrice(String content) {
|
||||
return parse(content, TOTAL_PRICE_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析单价
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseUnitPrice(String content) {
|
||||
return parse(content, UNIT_PRICE_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析面积
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Double parseArea(String content) {
|
||||
return parse(content, AREA_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析楼层
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseFloor(String content) {
|
||||
return parse(content, FLOOR_RESOLVERS)
|
||||
}
|
||||
/**
|
||||
* 解析楼高
|
||||
* @param content
|
||||
* @return
|
||||
*/
|
||||
static Integer parseHeight(String content) {
|
||||
return parse(content, HEIGHT_RESOLVERS)
|
||||
}
|
||||
|
||||
/**
|
||||
* 格式化小区名
|
||||
* @param name
|
||||
* @return
|
||||
*/
|
||||
static String formatName(String name) {
|
||||
name = name.replace("-商住", "").replace("-别墅", "")
|
||||
//汇福家园平安里(迁)2015-2017
|
||||
name = name.replaceAll("\\d\\d\\d\\d-\\d\\d\\d\\d", "")
|
||||
//国贸天悦一期2017
|
||||
name = name.replaceAll("\\d\\d\\d\\d\$", "")
|
||||
if (name ==~ /.+(\(|()[\d一二三四五六七八九十东南西北|\w]+(期|区)(\)|))$/) {
|
||||
name = name.replaceAll("\\(|(|\\)|)", '')
|
||||
} else if (name ==~ /.+(\(|()别墅(\)|))$/) {
|
||||
name = name.replaceAll("\\(|(|\\)|)", '')
|
||||
} else if (name ==~ /.+(\(|().+(\)|))$/) {
|
||||
name = name.replaceAll("(\\(|().+(\\)|))", '')
|
||||
} else if (name.contains("·") || (name.contains("-") && !(name ==~ /.*\d+-\d+.*/))) {
|
||||
println("---------------------------->" + name)
|
||||
name = name.replaceAll("·|-", '')
|
||||
println(name)
|
||||
}
|
||||
if (name ==~ /.{2,}路.{3,}$/ && !(name ==~ /.{2,}路.+号/) && !(name ==~ /.{2,}路.+弄/) && !(name ==~ /.*小区$/)) {
|
||||
name = name.replaceAll("^.{2,}路", "")
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
|
||||
static Object parse(String content, Map<Pattern, Closure> resolvers) {
|
||||
if (content != null) {
|
||||
content = content.replaceAll("\\s", "")
|
||||
}
|
||||
for (def resolver : resolvers.entrySet()) {
|
||||
String match = match(content, resolver.getKey())
|
||||
if (StringUtils.isNotBlank(match)) {
|
||||
match = match.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("两","2")
|
||||
return resolver.getValue()(match)
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配符合条件的字符串
|
||||
*
|
||||
* @param str
|
||||
* @param pattern
|
||||
* @return
|
||||
*/
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
static String parsePrice(String priceStr) {
|
||||
def pattern =Pattern.compile("\\d+(\\.\\d+)?万")
|
||||
String match = match(priceStr, pattern)
|
||||
}
|
||||
|
||||
static void main(String[] args) {
|
||||
println parsePrice("顶秀欣园西苑2室1厅1卫492.3万")
|
||||
}
|
||||
}
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
package com.fangjia.script.gj.crawl
|
||||
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.thirdPriceCrawl.Utils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class ZhugeFangDistCrawl {
|
||||
|
||||
public static final SHORT_DATE = Utils.getLastMonth();
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static main(args) {
|
||||
def task = new ZhugeFangDistCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/"
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.manage().deleteAllCookies()
|
||||
getRegions(driver, url)
|
||||
}
|
||||
|
||||
def getRegions(driver, url) {
|
||||
for(def i=1;i<=11;i++){
|
||||
def newUrl = url+"page${i}/"
|
||||
println newUrl
|
||||
def doc = getDocument(newUrl, driver)
|
||||
def total = parseList(doc)
|
||||
|
||||
// def next = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
// if(next){
|
||||
// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click()
|
||||
// }
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def parseList(doc){
|
||||
def city= "福州"
|
||||
def els = JQueryUtils.select("#listTableBox li", doc)
|
||||
if (els) {
|
||||
for(def e : els){
|
||||
def name = e.select(".house-name").text()
|
||||
def str = e.select(".house-adr").text()
|
||||
def address = null
|
||||
if(str){
|
||||
if(str.contains("]")){
|
||||
address = str.substring(str.indexOf("]")+1).trim()
|
||||
}else{
|
||||
address = str
|
||||
}
|
||||
}
|
||||
def regionStr = null
|
||||
if(str && str.contains("]") && str.contains("[")){
|
||||
regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]"))
|
||||
}
|
||||
|
||||
def region = null
|
||||
def block = null
|
||||
if(regionStr){
|
||||
def arrs = regionStr.split("-")
|
||||
if(arrs && arrs.size()>0){
|
||||
if(arrs.size()==2){
|
||||
block = arrs[1].trim()
|
||||
}
|
||||
if(arrs.size()>=1){
|
||||
region = arrs[0].trim()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def links = e.select(".house-name a")
|
||||
def href = null
|
||||
if(links){
|
||||
href = links[0].attr("href")
|
||||
}
|
||||
def priceStr = e.select(".price-number").text()
|
||||
def price = null
|
||||
if(priceStr && !priceStr.contains("暂无")){
|
||||
if(priceStr.contains("元")){
|
||||
priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim()
|
||||
}
|
||||
if(priceStr && priceStr.isNumber()){
|
||||
price = priceStr.toInteger()
|
||||
}
|
||||
}
|
||||
def obj = findByParams(city, region, name)
|
||||
if(obj){
|
||||
if(price){
|
||||
DBObject udata2 = new BasicDBObject()
|
||||
udata2.put("avg_price", price)
|
||||
udata2.put("m_date", new Date())
|
||||
println "[upeate] ${region} "+name
|
||||
thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2));
|
||||
}
|
||||
}else{
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", city)
|
||||
udata.put("region", region)
|
||||
udata.put("block", block)
|
||||
udata.put("url", href)
|
||||
udata.put("address", address)
|
||||
udata.put("name", name)
|
||||
udata.put("avg_price", price)
|
||||
udata.put("s_date", SHORT_DATE)
|
||||
udata.put("c_date", new Date())
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("source", "诸葛找房")
|
||||
thirdPartyColl.save(udata)
|
||||
println "[save] ${region} "+name+" "+priceStr
|
||||
}
|
||||
}
|
||||
}
|
||||
return els.size()
|
||||
}
|
||||
|
||||
|
||||
def findByParams(city, region, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("source", "诸葛找房")
|
||||
q.put("s_date", SHORT_DATE)
|
||||
return thirdPartyColl.findOne(q);
|
||||
}
|
||||
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
Thread.sleep(10000)
|
||||
def els = JQueryUtils.select("#listTableBox li", doc)
|
||||
if(els && els.size()>0){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
driver.manage().deleteAllCookies()
|
||||
println "[${error}] error: "+url
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=20){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
}
|
||||
+77
-52
@@ -55,7 +55,7 @@ class AddDistrictFieldValue {
|
||||
// }
|
||||
// })
|
||||
|
||||
// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
|
||||
def citys = ["重庆"]//CcbCommonUtils.citys;
|
||||
//第三方均价数据匹配
|
||||
// for(def city : citys){
|
||||
// def rets = findAllDataByCity(city)
|
||||
@@ -63,11 +63,11 @@ class AddDistrictFieldValue {
|
||||
// }
|
||||
|
||||
//第三方均价数据填补
|
||||
// for(def city : citys){
|
||||
//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||
// def rMap = getMatchDistrict(city)
|
||||
// createResultValue(dists, rMap)
|
||||
// }
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
|
||||
def rMap = getMatchDistrict(city)
|
||||
createResultValue(dists, rMap)
|
||||
}
|
||||
|
||||
// for(def city : citys){
|
||||
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
|
||||
@@ -103,52 +103,52 @@ class AddDistrictFieldValue {
|
||||
// }
|
||||
|
||||
|
||||
def citys = ["上海"]
|
||||
def updateIndex = 0
|
||||
def rets = []
|
||||
rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
|
||||
def total = dists.size()
|
||||
|
||||
for(def dist : dists){
|
||||
if(dist.household_count){
|
||||
def ret = []
|
||||
ret << dist._id.toString()
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.household_count
|
||||
ret << dist.loudong_count
|
||||
def details = findDistrictDetail(dist)
|
||||
if(details){
|
||||
for(def d : details){
|
||||
def region = RegionData.formatRegion(city, d.region)
|
||||
if(d.region == region){
|
||||
if(dist.household_count && dist.household_count==d.house_count){
|
||||
ret << d.loudong_count
|
||||
ret << "链家"
|
||||
// updateIndex++
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("household_count", d.house_count.toInteger())
|
||||
// udata.put("m_date", new Date())
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", dist._id)
|
||||
// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
|
||||
// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
|
||||
}
|
||||
// def citys = ["上海"]
|
||||
// def updateIndex = 0
|
||||
// def rets = []
|
||||
// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"]
|
||||
// for(def city : citys){
|
||||
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"])
|
||||
// def total = dists.size()
|
||||
//
|
||||
// for(def dist : dists){
|
||||
// if(dist.household_count){
|
||||
// def ret = []
|
||||
// ret << dist._id.toString()
|
||||
// ret << dist.city
|
||||
// ret << dist.region
|
||||
// ret << dist.name
|
||||
// ret << dist.household_count
|
||||
// ret << dist.loudong_count
|
||||
// def details = findDistrictDetail(dist)
|
||||
// if(details){
|
||||
// for(def d : details){
|
||||
// def region = RegionData.formatRegion(city, d.region)
|
||||
// if(d.region == region){
|
||||
// if(dist.household_count && dist.household_count==d.house_count){
|
||||
// ret << d.loudong_count
|
||||
// ret << "链家"
|
||||
//// updateIndex++
|
||||
//// def udata = new BasicDBObject()
|
||||
//// udata.put("household_count", d.house_count.toInteger())
|
||||
//// udata.put("m_date", new Date())
|
||||
//// def q = new BasicDBObject()
|
||||
//// q.put("_id", dist._id)
|
||||
//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count
|
||||
//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }else{
|
||||
// ret << ""
|
||||
// ret << ""
|
||||
// }
|
||||
//
|
||||
// rets << ret
|
||||
// }
|
||||
// }
|
||||
// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据")
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
@@ -303,6 +303,14 @@ class AddDistrictFieldValue {
|
||||
def createResultValue(dists, rMap){
|
||||
def updateIndex = 0
|
||||
def total = dists.size()
|
||||
def addressIndex = 0
|
||||
def allAreaIndex = 0
|
||||
def buildAreaIndex = 0
|
||||
def virescence_percent_index = 0
|
||||
def cubage_percent_index = 0
|
||||
def finish_building_date_index = 0
|
||||
def household_count_index = 0
|
||||
def car_site_count_index = 0
|
||||
for(def d : dists){
|
||||
def key = d.city +"#"+d.region+"#"+d.name
|
||||
def sourceMap = rMap.get(key)
|
||||
@@ -311,12 +319,14 @@ class AddDistrictFieldValue {
|
||||
if(!d.address){
|
||||
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
|
||||
if(lastValue){
|
||||
addressIndex++
|
||||
udata.put("address", lastValue)
|
||||
}
|
||||
}
|
||||
if(!d.all_area){
|
||||
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
allAreaIndex++
|
||||
udata.put("all_area", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -324,6 +334,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.build_area){
|
||||
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
|
||||
if(lastValue){
|
||||
buildAreaIndex++
|
||||
udata.put("build_area", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -331,6 +342,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.virescence_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
virescence_percent_index ++
|
||||
udata.put("virescence_percent", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -338,6 +350,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.cubage_percent){
|
||||
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
|
||||
if(lastValue){
|
||||
cubage_percent_index ++
|
||||
udata.put("cubage_percent", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -345,6 +358,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.finish_building_date){
|
||||
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
|
||||
if(lastValue){
|
||||
finish_building_date_index ++
|
||||
udata.put("finish_building_date", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -352,6 +366,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.household_count){
|
||||
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
|
||||
if(lastValue){
|
||||
household_count_index ++
|
||||
udata.put("household_count", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -359,6 +374,7 @@ class AddDistrictFieldValue {
|
||||
if(!d.car_site_count){
|
||||
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
|
||||
if(lastValue){
|
||||
car_site_count_index ++
|
||||
udata.put("car_site_count", lastValue)
|
||||
}
|
||||
}
|
||||
@@ -373,6 +389,15 @@ class AddDistrictFieldValue {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "addressIndex "+ addressIndex
|
||||
println "allAreaIndex "+ allAreaIndex
|
||||
println "buildAreaIndex "+ buildAreaIndex
|
||||
println "virescence_percent_index "+ virescence_percent_index
|
||||
println "cubage_percent_index "+ cubage_percent_index
|
||||
println "finish_building_date_index "+ finish_building_date_index
|
||||
println "household_count_index "+ household_count_index
|
||||
println "car_site_count_index "+ car_site_count_index
|
||||
}
|
||||
|
||||
def getValuesByField(sourceMap, field, estateType){
|
||||
|
||||
+19
-2
@@ -38,8 +38,8 @@ class FangjiaClientUtils {
|
||||
}
|
||||
|
||||
public static def sendDistrictPriceCalculate(def city, def region, def name, def count){
|
||||
def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true"
|
||||
def content = null;
|
||||
try{
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
@@ -83,6 +83,23 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def arroundPrice(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
// def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8")
|
||||
def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100"
|
||||
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
|
||||
public static Map<String, Object> JsonToMap(JSONObject json) {
|
||||
Map<String, Object> columnValMap = new HashMap<String, Object>();
|
||||
|
||||
+2
-1
@@ -172,7 +172,8 @@ class NameProcess {
|
||||
if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){
|
||||
return null
|
||||
}
|
||||
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){
|
||||
//|| name.endsWith("组")
|
||||
if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){
|
||||
return null
|
||||
}
|
||||
if(name.contains("、")||name.contains("|")||name.contains(".")){
|
||||
|
||||
+17
-5
@@ -58,6 +58,8 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new DistrictIndexBatch(), args);
|
||||
}
|
||||
@@ -69,9 +71,10 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
def run(){
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]
|
||||
//def citys = CommonDatautils.findAllCity();
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"])
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"])
|
||||
def code = cityCodeDatas.get(city)
|
||||
def city_avg_total_price = null
|
||||
if(city!="ÉϺ£"){
|
||||
@@ -131,7 +134,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
map.total_price = total_price
|
||||
//map.comp_value = potentiality
|
||||
//map.avg_total_price = total_price_index
|
||||
map.month = "2022-03"
|
||||
map.month = month
|
||||
map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
//datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality]
|
||||
@@ -229,7 +232,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
|
||||
map.comp_value = potentiality
|
||||
map.avg_total_price = total_price_index
|
||||
map.month = "2022-03"
|
||||
map.month = month
|
||||
map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T")
|
||||
|
||||
map.finish_date = year
|
||||
@@ -240,6 +243,15 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
if(rMap.sell_avg_price && sqmetre_rent){
|
||||
map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger()
|
||||
}
|
||||
if(d.household_count){
|
||||
map.household_count = d.household_count
|
||||
}
|
||||
if(d.lat){
|
||||
def lMap = [:]
|
||||
lMap.lon = d.lng/10000000000.0
|
||||
lMap.lat = d.lat/10000000000.0
|
||||
map.location = lMap
|
||||
}
|
||||
def newUrl = targetUrl +"_doc/"+map.id
|
||||
httpSaveData(newUrl, map)
|
||||
|
||||
@@ -560,7 +572,7 @@ class DistrictIndexBatch extends BaseBatch{
|
||||
return datas
|
||||
}
|
||||
|
||||
def sortByDistance(d, list){
|
||||
def sortByDistance(d, list){
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
|
||||
+123
@@ -0,0 +1,123 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
class DistrictMapMatch {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictMapMatch()
|
||||
task.mapMatch()
|
||||
}
|
||||
|
||||
|
||||
def mapMatch(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d->
|
||||
if(!d['H']){
|
||||
index++
|
||||
def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C'])
|
||||
def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// if (!map || !map.lat) {
|
||||
// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']);
|
||||
// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets);
|
||||
// }
|
||||
if(map){
|
||||
d['F'] = map.id
|
||||
if(map.adname){
|
||||
d['G'] = map.adname
|
||||
}else if(map.region){
|
||||
d['G'] = map.region
|
||||
}
|
||||
d['H'] = map.name
|
||||
d['I'] = map.address
|
||||
if(map.type){
|
||||
d['J'] = map.type
|
||||
}else if(map.estateType){
|
||||
d['J'] = map.estateType
|
||||
}
|
||||
d['K'] = map.lng
|
||||
d['L'] = map.lat
|
||||
d['M'] = map.score
|
||||
d['N'] = map.source
|
||||
}
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
def findStandardObject2(def city, def region, def name, def address, def rets) {
|
||||
if (rets) {
|
||||
def bestObj = [:]
|
||||
if (rets.size() >= 1) {
|
||||
for (def mapObj : rets) {
|
||||
def score = 0
|
||||
if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){
|
||||
score = 0.2
|
||||
}
|
||||
if(mapObj.name && name){
|
||||
def newName = mapObj.name;
|
||||
def names = []
|
||||
if(mapObj.name && mapObj.name.indexOf("(")>-1){
|
||||
def name1 = mapObj.name.substring(0, mapObj.name.indexOf("("))
|
||||
names << name1
|
||||
def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")"))
|
||||
names << name2
|
||||
}else{
|
||||
names << newName
|
||||
}
|
||||
def maxNameScore = 0
|
||||
for(def n : names){
|
||||
double score1 = CosineSimilarAlgorithm.getSimilarity(n, name);
|
||||
if(score1 && score1>0){
|
||||
def nameScore = (score1 * 4)/10
|
||||
if(nameScore > maxNameScore){
|
||||
maxNameScore = nameScore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(maxNameScore){
|
||||
try{
|
||||
maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore)
|
||||
}catch(Exception e){
|
||||
maxNameScore = 0
|
||||
}
|
||||
|
||||
}
|
||||
score = maxNameScore + score;
|
||||
}
|
||||
|
||||
if(mapObj.address && address){
|
||||
double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address);
|
||||
if(addressScore && addressScore>0){
|
||||
try{
|
||||
addressScore = DataHandleUtils.getDoubleRound(2, addressScore)
|
||||
}catch(Exception e){
|
||||
addressScore = 0
|
||||
}
|
||||
score = addressScore*4/10 + score
|
||||
}
|
||||
}
|
||||
|
||||
if(!bestObj){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}else{
|
||||
if(score > bestObj.score){
|
||||
mapObj.score = score
|
||||
bestObj = mapObj
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return bestObj
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
}
|
||||
+402
@@ -0,0 +1,402 @@
|
||||
package com.fangjia.script.gj.meituan
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.gj.paimai.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class MatchTest {
|
||||
static main(args) {
|
||||
def task = new MatchTest()
|
||||
// task.run()
|
||||
// task.addNum()
|
||||
// task.dongguang()
|
||||
// task.meituanDel()
|
||||
|
||||
task.addLevel()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addLevel() {
|
||||
def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]//
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def cityLevelMap = [:]
|
||||
for(def city : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def datas = addPriceData(dists, priceMap);
|
||||
datas = calcuteLevel(datas);
|
||||
cityLevelMap.put(city, ["price":priceMap, "level":datas])
|
||||
println city +" 楼盘评级同步完成"
|
||||
}
|
||||
|
||||
|
||||
def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
|
||||
def name = row['D']
|
||||
def address = row['E']
|
||||
def city = RegionData.formatCity(row['A'])
|
||||
|
||||
if(name && !row['O']){
|
||||
if(city){
|
||||
def map = cityLevelMap.get(city)
|
||||
def rMap = null
|
||||
if(map){
|
||||
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
if(rMap.rank){
|
||||
row['Q'] = rMap.rank
|
||||
}
|
||||
// if(rMap.level){
|
||||
// row['Q'] = rMap.level
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def meituanDel(){
|
||||
def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx"
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { row ->
|
||||
def flag = row['O']
|
||||
if(flag){
|
||||
def city = row['F']
|
||||
def region = row['G']
|
||||
def name = nameFileter(row['H'])
|
||||
def obj = null
|
||||
if(city == "东莞"){
|
||||
obj = CommonDatautils.findDistrict(city, null, name, "district")
|
||||
}else{
|
||||
obj = CommonDatautils.findDistrict(city, region, name, "district")
|
||||
}
|
||||
if(obj){
|
||||
index++
|
||||
println index +" "+ city +' '+ region+" "+name
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_cat", 11)
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(obj._id.toString()))
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def dongguang(){
|
||||
def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx
|
||||
ExcelUtils.modify(path, { row ->
|
||||
def region = null
|
||||
if(row['B'] && row['B']=="NULL" ){
|
||||
region = null
|
||||
}
|
||||
def name = nameFileter(row['H'])
|
||||
def address = row['E']
|
||||
if (name && name.contains("湖畔新家")) {
|
||||
|
||||
def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district")
|
||||
if(obj){
|
||||
def json = GaoDeApiUtils.geo(row['F'], address)
|
||||
if(json && json.count>=1){
|
||||
def m = json.geocodes[0]
|
||||
if(m){
|
||||
def twinshop = getStreetInfo(m.location)
|
||||
if(twinshop){
|
||||
def newRegion = RegionData.formatRegion("东莞",twinshop)
|
||||
if(newRegion){
|
||||
def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district")
|
||||
if(dbObj){
|
||||
println "delete"
|
||||
DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id))
|
||||
}else{
|
||||
println "update"
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
// println address +" "+twinshop
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def getStreetInfo(def glocation){
|
||||
def township = null
|
||||
if(glocation){
|
||||
def jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
return township;
|
||||
}
|
||||
|
||||
def addNum(){
|
||||
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx
|
||||
def index = 0
|
||||
ExcelUtils.modify(path, { row ->
|
||||
index++
|
||||
row['A'] = index
|
||||
},0)
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]//
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
def cityLevelMap = [:]
|
||||
def radioMap = [:]
|
||||
for(def city : citys){
|
||||
|
||||
// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1));
|
||||
// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay)
|
||||
// radioMap.put(city, avg_radio)
|
||||
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"])
|
||||
def priceMap = CommonDatautils.queryDistrictPrice(city, weekend)
|
||||
def datas = addPriceData(dists, priceMap);
|
||||
datas = calcuteLevel(datas);
|
||||
cityLevelMap.put(city, ["price":priceMap, "level":datas])
|
||||
println city +" 楼盘评级同步完成"
|
||||
|
||||
}
|
||||
|
||||
|
||||
def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def region = null
|
||||
if(row['C'] && row['C']=="NULL" ){
|
||||
region = null
|
||||
}
|
||||
def name = row['D']
|
||||
def address = row['F']
|
||||
def city = RegionData.formatCity(row['B'])
|
||||
|
||||
if(name && !row['N']){
|
||||
if(city){
|
||||
def map = cityLevelMap.get(city)
|
||||
def rMap = null
|
||||
if(map){
|
||||
rMap = getArroundDistrictPriceByAddress(city, address, map.level)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
if(rMap.rank){
|
||||
row['Q'] = rMap.rank
|
||||
}
|
||||
if(rMap.level){
|
||||
row['P'] = rMap.level
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (name && row['N']) {
|
||||
//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name)
|
||||
// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null)
|
||||
// if (jsonObject != null && jsonObject.precise != -1) {
|
||||
// row['F'] = jsonObject.get("city")
|
||||
// row['G'] = jsonObject.get("district")
|
||||
// row['H'] = jsonObject.get("name")
|
||||
// row['I'] = jsonObject.get("propertyType")
|
||||
// row['J'] = jsonObject.get("map")
|
||||
// row['K'] = jsonObject.get("lng")
|
||||
// row['L'] = jsonObject.get("lat")
|
||||
// }
|
||||
|
||||
if(row['G'] && row['H'] && row['I']){
|
||||
// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H'])
|
||||
// if(priceObj){
|
||||
// row['M'] = priceObj.avg_price
|
||||
// }
|
||||
// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H'])
|
||||
// if(sqmetreRentPrice){
|
||||
// row['N'] = sqmetreRentPrice
|
||||
// }
|
||||
|
||||
if(row['N'] && !row['O']){
|
||||
def avg_rent_radio = radioMap.get(city)
|
||||
row['O'] = (int)(row['N']/avg_rent_radio)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def getArroundDistrictPriceByAddress(def city, def address, def levels){
|
||||
def json = GaoDeApiUtils.geo(city, address)
|
||||
def location = null
|
||||
def detail = new BasicDBObject()
|
||||
def level = null
|
||||
def rank = null
|
||||
if(json && json.count>=1){
|
||||
def m = json.geocodes[0]
|
||||
if(m){
|
||||
location = m.location
|
||||
level = m.level
|
||||
}
|
||||
}
|
||||
if(location){
|
||||
def arrs = location.split(",")
|
||||
detail.lng = arrs[0].toString().toDouble()*10000000000L
|
||||
detail.lat = arrs[1].toString().toDouble()*10000000000L
|
||||
}
|
||||
if(detail.lng){
|
||||
detail.city = city
|
||||
|
||||
List<DBObject> list = DbUtils.findAroundDistrictByRadius(detail, 3000)
|
||||
if(list && list.size()>10){
|
||||
list = sortByDistance(detail, list);
|
||||
list = list[0..9]
|
||||
}
|
||||
def arroundAvgPrice = getArroundAvgPrice(list)
|
||||
if(arroundAvgPrice){
|
||||
|
||||
def mod = Integer.MAX_VALUE;
|
||||
for(def d : levels){
|
||||
def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice)
|
||||
if(newMod<mod){
|
||||
mod = newMod
|
||||
rank = d.level
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
def m = [:]
|
||||
m.level = level
|
||||
m.rank = rank
|
||||
return m
|
||||
}
|
||||
|
||||
|
||||
def sortByDistance(d, list){
|
||||
for(def nb : list){
|
||||
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
|
||||
nb.distance = distance
|
||||
}
|
||||
//排序
|
||||
Collections.sort(list, new Comparator() {
|
||||
@Override
|
||||
int compare(Object o1, Object o2) {
|
||||
if(o1.distance > o2.distance)
|
||||
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
|
||||
if(o1.distance == o2.distance)
|
||||
return 0;
|
||||
return -1;
|
||||
}
|
||||
})
|
||||
return list;
|
||||
}
|
||||
|
||||
def addPriceData(dists, priceMap){
|
||||
def newLists = []
|
||||
for(def d : dists){
|
||||
def price = priceMap.get(d.city+"##"+d.region+"##"+d.name);
|
||||
if(price){
|
||||
d.avg_price = price;
|
||||
newLists << d;
|
||||
}
|
||||
}
|
||||
if(newLists){
|
||||
newLists.sort {a,b->
|
||||
def v1 = a.avg_price
|
||||
def v2 = b.avg_price
|
||||
return v1.compareTo(v2)
|
||||
}
|
||||
}
|
||||
return newLists;
|
||||
}
|
||||
|
||||
def calcuteLevel(def datas){
|
||||
def total = datas.size()
|
||||
def v = (int)(total/10)
|
||||
def start = 0;
|
||||
def end = 0;
|
||||
def map = [:]
|
||||
for(int i=1;i<=10;i++){
|
||||
end = end + v;
|
||||
if(i == 10){
|
||||
map.put(i, [start, total])
|
||||
}else{
|
||||
map.put(i, [start, end])
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
for(def d=0;d<datas.size();d++){
|
||||
def dist = datas[d]
|
||||
|
||||
for (def o in map) {
|
||||
def ranges = o.value
|
||||
def m = ranges[0]
|
||||
def n = ranges[1]
|
||||
if(d==0){
|
||||
if(d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}else{
|
||||
if(d>m && d<=n){
|
||||
dist.level = 10 - o.key + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getArroundAvgPrice(def list){
|
||||
def prices = []
|
||||
for(def l : list){
|
||||
def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name)
|
||||
if(priceObj){
|
||||
prices << priceObj.avg_price
|
||||
}
|
||||
}
|
||||
if(prices){
|
||||
def avg_price = (prices.sum()/prices.size()).toInteger()
|
||||
return avg_price;
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def nameFileter(def name){
|
||||
name = name.replace("·","").replace(".","").replace("|","")
|
||||
name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期")
|
||||
name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区")
|
||||
return name;
|
||||
}
|
||||
|
||||
}
|
||||
+48
-26
@@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome {
|
||||
|
||||
static main(args) {
|
||||
def task = new BeiKeCrawlByChrome();
|
||||
// task.run()
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
}
|
||||
@@ -45,9 +45,21 @@ public class BeiKeCrawlByChrome {
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def cityMap = getAllCityUrls(driver)
|
||||
def citys = getAllCityUrls(driver)
|
||||
citys = citys.reverse()
|
||||
def start = 0
|
||||
// for(def i=0;i<citys.size();i++){
|
||||
// def m = citys[i]
|
||||
// println i+" "+ m.city
|
||||
// if(m.city=="天津"){
|
||||
// start = i
|
||||
// break;
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// citys = citys[(start+1)..(citys.size()-1)]
|
||||
getLogin(driver)
|
||||
crawlData(driver, cityMap)
|
||||
crawlData(driver, citys)
|
||||
|
||||
|
||||
}
|
||||
@@ -85,13 +97,13 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
}
|
||||
|
||||
def crawlData(driver, cityMap){
|
||||
def crawlData(driver, citys){
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
cityMap.each{city, cityUrl->
|
||||
for(def m : citys){
|
||||
//def detailUrls = getPageUrls(driver, cityUrl)
|
||||
def detailUrls = getPageDeail(driver, city, cityUrl)
|
||||
println cityUrl +" 详情链接:"+detailUrls.size()
|
||||
def detailUrls = getPageDeail(driver, m.city, m.url)
|
||||
println m.city +" 详情链接:"+detailUrls.size()
|
||||
// crawlDetail(driver, DEAL_PRICE_COLL, detailUrls)
|
||||
}
|
||||
}
|
||||
@@ -110,7 +122,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
@@ -141,7 +153,7 @@ public class BeiKeCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -150,6 +162,10 @@ public class BeiKeCrawlByChrome {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -177,12 +193,12 @@ public class BeiKeCrawlByChrome {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
if (areaStr && areaStr.isNumber()) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
@@ -199,7 +215,8 @@ public class BeiKeCrawlByChrome {
|
||||
detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd")
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
def dbo = findTrade(detail)
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
def dbo = findTrade(detailUrl)
|
||||
if (!dbo) {
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
def houseInfos = houseInfo.split("\\|")
|
||||
@@ -212,7 +229,7 @@ public class BeiKeCrawlByChrome {
|
||||
detail.height = floorStr.toInteger()
|
||||
}
|
||||
}
|
||||
detail.url = e.select(".title a").attr("href")
|
||||
detail.url = detailUrl
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
@@ -297,8 +314,8 @@ public class BeiKeCrawlByChrome {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅"))
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫"))
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger()
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger()
|
||||
}
|
||||
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
|
||||
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
|
||||
@@ -320,13 +337,14 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
|
||||
def findTrade(def detail){
|
||||
def findTrade(def detailUrl){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", detail.city)
|
||||
q.put("region", detail.region)
|
||||
q.put("district_name", detail.district_name)
|
||||
q.put("area", detail.area)
|
||||
q.put("total_price", detail.total_price)
|
||||
// q.put("region", detail.region)
|
||||
// q.put("district_name", detail.district_name)
|
||||
// q.put("area", detail.area)
|
||||
// q.put("total_price", detail.total_price)
|
||||
q.put("url", detailUrl)
|
||||
q.put("trade_date", detail.trade_date)
|
||||
q.put("source", detail.source)
|
||||
def dbo = DEAL_PRICE_COLL.findOne(q)
|
||||
@@ -337,17 +355,21 @@ public class BeiKeCrawlByChrome {
|
||||
driver.get("https://www.ke.com/city/")
|
||||
Thread.sleep(8000)
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc)
|
||||
def list = []
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(!e.text().contains("南京")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
// if(e.text().contains("福州")){
|
||||
def cityMap = [:]
|
||||
cityMap.city = e.text()
|
||||
cityMap.url = "https:"+href
|
||||
list << cityMap
|
||||
//cityMap.put(e.text(), "https:"+href)
|
||||
// }
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
return list
|
||||
}
|
||||
|
||||
def getLogin(driver){
|
||||
|
||||
+285
-124
@@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver
|
||||
import org.openqa.selenium.support.ui.ExpectedConditions
|
||||
import org.openqa.selenium.support.ui.WebDriverWait
|
||||
|
||||
import javax.swing.JApplet
|
||||
import java.text.DateFormat
|
||||
import java.util.function.Consumer
|
||||
|
||||
@@ -41,45 +42,138 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
|
||||
|
||||
def doc = getDocument();
|
||||
def detailUrls = getDetailLinks(doc);
|
||||
def rets = batchParse(detailUrls);
|
||||
saveToDB(rets);
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def proviceMap = getDocument(driver);
|
||||
def citys = getCity(driver,proviceMap)
|
||||
def detailUrls = getDetailLinks(driver, citys);
|
||||
|
||||
|
||||
def getDetailLinks(doc){
|
||||
def urls = []
|
||||
|
||||
def getDetailLinks(driver, cityList){
|
||||
|
||||
def zxcitys = ["天津市","北京市","上海市","重庆市"]
|
||||
for(def cityMap : cityList){
|
||||
def citys = cityMap.citys
|
||||
|
||||
citys.each{city,id->
|
||||
def urls = []
|
||||
if(!zxcitys.contains(cityMap.provice)){
|
||||
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
|
||||
getDocument(url, driver)
|
||||
}else{
|
||||
def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport="
|
||||
getDocument(url, driver)
|
||||
}
|
||||
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def count = JQueryUtils.selectText(".count", doc)
|
||||
def totalPage = getTotalPage(count, 12)
|
||||
if(totalPage){
|
||||
for(def i=1;i<=totalPage;i++){
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
|
||||
if(eles){
|
||||
for(def el :eles){
|
||||
def href = el.attr("href")
|
||||
def map = [:]
|
||||
map.province = cityMap.provice
|
||||
map.city = city
|
||||
map.url = "https://paimai.caa123.org.cn"+href
|
||||
urls << map
|
||||
}
|
||||
}
|
||||
try{
|
||||
driver.findElement(By.className("next")).click()
|
||||
}catch(Exception e){
|
||||
Thread.sleep(8000)
|
||||
driver.findElement(By.className("next")).click()
|
||||
}
|
||||
}
|
||||
|
||||
def rets = batchParse(urls, driver);
|
||||
saveToDB(rets);
|
||||
println city +" "+urls.size()
|
||||
}
|
||||
}
|
||||
println cityMap.provice +" 完成"
|
||||
|
||||
}
|
||||
|
||||
//return urls;
|
||||
}
|
||||
|
||||
def getTotalPage(def total, def pageSize){
|
||||
if(total){
|
||||
def mode = total.toInteger()/pageSize
|
||||
if(mode > mode.toInteger()){
|
||||
mode = mode.toInteger() +1
|
||||
}
|
||||
return mode;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
def getCity(driver, proviceMap){
|
||||
def rows = []
|
||||
def zxcitys = ["天津市","北京市","上海市","重庆市"]
|
||||
proviceMap.each {k,v->
|
||||
if(!zxcitys.contains(k)){
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc)
|
||||
def cityMap = [:]
|
||||
for(def c : cityEles){
|
||||
def cityid = c.attr("data-cityid")
|
||||
def city = c.text()
|
||||
if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){
|
||||
cityMap.put(city, cityid)
|
||||
}
|
||||
|
||||
}
|
||||
def map = [:]
|
||||
map.provice = k
|
||||
map.provice_id = v
|
||||
map.citys = cityMap
|
||||
rows << map
|
||||
}else{
|
||||
def map = [:]
|
||||
map.provice = k
|
||||
map.provice_id = v
|
||||
def cityMap = [:]
|
||||
cityMap.put(k, v)
|
||||
map.citys = cityMap
|
||||
rows << map
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
def getDocument(driver){
|
||||
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport=");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def provinceMap = [:]
|
||||
if(doc){
|
||||
def eles = JQueryUtils.select(".lots-list .lot-li-a", doc)
|
||||
if(eles){
|
||||
for(def el :eles){
|
||||
def href = el.attr("href")
|
||||
urls << "https://paimai.caa123.org.cn"+href
|
||||
def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc)
|
||||
for(def ele : proviceEles){
|
||||
def provice = ele.text()
|
||||
def value = ele.attr("data-proviceid")
|
||||
if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){
|
||||
provinceMap.put(provice, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
return urls;
|
||||
return provinceMap;
|
||||
}
|
||||
|
||||
def getDocument(){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted=");
|
||||
WebDriverWait wait = new WebDriverWait(driver,20)
|
||||
wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list"))));
|
||||
def batchParse(def urls, driver){
|
||||
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
def batchParse(def urls){
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def ret = []
|
||||
for(def url :urls){
|
||||
driver.get(url);
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def res = parse(url, doc);
|
||||
for(def map :urls){
|
||||
def doc = getDocument(map.url, driver)
|
||||
def res = parse(map.province, map.city, map.url, doc);
|
||||
if(res){
|
||||
ret << res
|
||||
}
|
||||
@@ -87,111 +181,167 @@ def batchParse(def urls){
|
||||
return ret;
|
||||
}
|
||||
|
||||
def parse(url,doc){
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
|
||||
def parse(province, city, url,doc){
|
||||
if(doc){
|
||||
def result = [:]
|
||||
// def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||
// result.address = address
|
||||
if(result.address){
|
||||
def rMap = OpenApiUtils.matchDistrict(result.address)
|
||||
if(rMap) {
|
||||
result.matchCity = rMap.matchCity
|
||||
result.matchRegion = rMap.matchRegion
|
||||
result.matchName= rMap.matchName
|
||||
result.matchAddress = rMap.matchAddress
|
||||
result.province = province
|
||||
result.city = city
|
||||
def address = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||
result.address = address
|
||||
if(result.address && !result.address.contains("测试")) {
|
||||
def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
|
||||
if(finishDateStr) {
|
||||
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss");
|
||||
}
|
||||
result.status = JQueryUtils.selectText([".bd_state"], doc)
|
||||
result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc)
|
||||
result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc)
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",","")
|
||||
result.tradePrice = curPriceStr.toInteger()
|
||||
}
|
||||
def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc)
|
||||
if (finishDateStr) {
|
||||
try{
|
||||
result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm");
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
|
||||
if(evaluationPriceStr) {
|
||||
evaluationPriceStr = evaluationPriceStr.replace(",", "")
|
||||
if(evaluationPriceStr.isInteger()) {
|
||||
result.evaluationPrice = evaluationPriceStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
|
||||
if(depositPriceStr) {
|
||||
depositPriceStr = depositPriceStr.replace(",", "")
|
||||
if(depositPriceStr.isInteger()) {
|
||||
result.depositPrice = depositPriceStr.toInteger()
|
||||
def curPriceStr = JQueryUtils.selectText([".colorred"], doc)
|
||||
if (curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "")
|
||||
result.tradePrice = curPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
|
||||
if(addPriceStr) {
|
||||
addPriceStr = addPriceStr.replace(",", "")
|
||||
if(addPriceStr.isInteger()) {
|
||||
result.addPrice = addPriceStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
|
||||
if(startPriceStr) {
|
||||
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
|
||||
if(startPriceStr.isInteger()) {
|
||||
result.startPrice = startPriceStr.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc)
|
||||
result.retain = JQueryUtils.selectText(["#retain"], doc)
|
||||
|
||||
def imgs =JQueryUtils.select(".sf-pic-slide img", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
for(def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if(href.contains("http")) {
|
||||
imgLinks << href
|
||||
}else {
|
||||
imgLinks << url.substring(0, url.indexOf(":")+1)+href
|
||||
def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc)
|
||||
if (evaluationPriceStr) {
|
||||
evaluationPriceStr = evaluationPriceStr.replace(",", "")
|
||||
if (evaluationPriceStr.isNumber()) {
|
||||
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
def desc = JQueryUtils.selectText(["#J_desc"], doc)
|
||||
if(!desc){
|
||||
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||
}
|
||||
if(desc) {
|
||||
if(desc.contains("别墅")) {
|
||||
result.address = desc.substring(0, desc.indexOf("别墅")+2)
|
||||
}else if(desc.contains("号楼")) {
|
||||
result.address = desc.substring(0, desc.indexOf("号楼")+2)
|
||||
}else if(desc.contains("房屋")) {
|
||||
result.address = desc.substring(0, desc.indexOf("房屋")+2)
|
||||
def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc)
|
||||
if (depositPriceStr) {
|
||||
depositPriceStr = depositPriceStr.replace(",", "")
|
||||
if (depositPriceStr.isNumber()) {
|
||||
result.depositPrice = depositPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def descr = JQueryUtils.selectText(["body"], doc)
|
||||
if(descr) {
|
||||
def area = getAreaNumber(descr, 5000);
|
||||
if(area){
|
||||
result.area = area.toDouble()
|
||||
def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc)
|
||||
if (addPriceStr) {
|
||||
addPriceStr = addPriceStr.replace(",", "")
|
||||
if (addPriceStr.isNumber()) {
|
||||
result.addPrice = addPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
if(result.area && result.tradePrice){
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/result.area)
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
|
||||
result.url = url
|
||||
return result;
|
||||
def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc)
|
||||
if (startPriceStr) {
|
||||
startPriceStr = startPriceStr.replace(",", "").replace("元", "")
|
||||
if (startPriceStr.isNumber()) {
|
||||
result.startPrice = startPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc)
|
||||
result.retain = JQueryUtils.selectText(["#retain"], doc)
|
||||
result.lookCount = JQueryUtils.selectText(".look_count span", doc)
|
||||
|
||||
def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc)
|
||||
if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){
|
||||
startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim()
|
||||
}
|
||||
|
||||
try{
|
||||
result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim())
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
def imgs = JQueryUtils.select(".sf-pic-slide img", doc)
|
||||
if (imgs) {
|
||||
def imgLinks = []
|
||||
for (def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if (href.contains("http")) {
|
||||
imgLinks << href
|
||||
} else {
|
||||
imgLinks << url.substring(0, url.indexOf(":") + 1) + href
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
def desc = JQueryUtils.selectText(["#J_desc"], doc)
|
||||
if (!desc) {
|
||||
desc = JQueryUtils.selectText(["#pro_bid_name"], doc)
|
||||
}
|
||||
if (desc) {
|
||||
if (desc.contains("别墅")) {
|
||||
result.address = desc.substring(0, desc.indexOf("别墅") + 2)
|
||||
} else if (desc.contains("号楼")) {
|
||||
result.address = desc.substring(0, desc.indexOf("号楼") + 2)
|
||||
} else if (desc.contains("房屋")) {
|
||||
result.address = desc.substring(0, desc.indexOf("房屋") + 2)
|
||||
}
|
||||
}
|
||||
|
||||
def descr = JQueryUtils.selectText(["body"], doc)
|
||||
if (descr) {
|
||||
def area = getAreaNumber(descr, 5000);
|
||||
if (area) {
|
||||
result.area = area.toDouble()
|
||||
}
|
||||
}
|
||||
if (result.area && result.tradePrice) {
|
||||
result.tradeAvgPrice = (int) (result.tradePrice / result.area)
|
||||
}
|
||||
|
||||
if(result.finishDate){
|
||||
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
||||
}
|
||||
if(result.startDate){
|
||||
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
|
||||
}
|
||||
|
||||
result.content = DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString()
|
||||
result.url = url
|
||||
return result;
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
@@ -207,11 +357,13 @@ def saveToDB(rets){
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
obj.source= "中拍网"
|
||||
if(dbo){
|
||||
obj.remove("url")
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.save(obj)
|
||||
}
|
||||
}
|
||||
@@ -229,16 +381,25 @@ def getAreaNumber(input, maxArea) {
|
||||
def area = null;
|
||||
|
||||
for (def val : list) {
|
||||
if(val.toString().contains(" ")){
|
||||
val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size())
|
||||
}
|
||||
if(val.contains("室")){
|
||||
val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size())
|
||||
}
|
||||
def v = val.replace("ƽ·½Ã×", "").replace("©O", "").replace("ƽ", "").replace("·½Ã×", "").replace(",", "").replace("m2", "");
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
if(v && v.toString().isNumber()){
|
||||
if (maxArea) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (area) {
|
||||
|
||||
@@ -0,0 +1,311 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import org.apache.commons.codec.binary.Base64
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import sun.font.Font2D
|
||||
import sun.font.Font2DHandle
|
||||
import sun.font.TrueTypeFont
|
||||
import sun.font.TrueTypeGlyphMapper
|
||||
|
||||
import java.awt.Font
|
||||
import java.awt.font.FontRenderContext
|
||||
import java.awt.font.GlyphVector
|
||||
import java.awt.geom.AffineTransform
|
||||
import java.lang.reflect.Field
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DataUtil {
|
||||
|
||||
|
||||
static def checkMinDate(curDate, checkDate){
|
||||
if(curDate){
|
||||
def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
|
||||
if(curDate.getTime()<minDate.getTime()){
|
||||
return null
|
||||
}
|
||||
}
|
||||
return curDate
|
||||
}
|
||||
|
||||
static def checkMaxDate(curDate, checkDate){
|
||||
if(curDate){
|
||||
def maxDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd")
|
||||
if(curDate.getTime()>maxDate.getTime()){
|
||||
return null
|
||||
}
|
||||
}
|
||||
return curDate
|
||||
}
|
||||
static def getAreaNumber(def input, def maxArea) {
|
||||
Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)')
|
||||
Matcher matcher = pattern.matcher(input)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() > 0) {
|
||||
def area = null
|
||||
|
||||
for (def val : list) {
|
||||
def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "")
|
||||
if (maxArea) {
|
||||
if (v.isDouble()) {
|
||||
if (v.toDouble() < maxArea) {
|
||||
area = v
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
area = v
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (area) {
|
||||
return area
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def getPaiMaiType(def s) {
|
||||
if (s) {
|
||||
if (s.toString().contains("结束")) {
|
||||
return "已完成"
|
||||
} else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) {
|
||||
return "进行中"
|
||||
} else if (s.toString().contains("即将开始")) {
|
||||
return "即将开始"
|
||||
} else if (s.toString().contains("流拍")) {
|
||||
return "流拍"
|
||||
} else if (s.toString().contains("撤回")) {
|
||||
return "撤回"
|
||||
} else if (s.toString().contains("暂缓")) {
|
||||
return "暂缓"
|
||||
} else if (s.toString().contains("中止")) {
|
||||
return "中止"
|
||||
} else if (s.toString().contains("失败")) {
|
||||
return "失败"
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
static def getDateRange(def s) {
|
||||
s = s.replace("起", "")
|
||||
Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
def str = matcher.group()
|
||||
list.add(str)
|
||||
// if(str) {
|
||||
// def indx1 = str.indexOf("年")
|
||||
// def indx2 = str.indexOf("月")
|
||||
// def month = str.substring(indx1+1, indx2);
|
||||
// if(month.toInteger()<12) {
|
||||
// list.add(str);
|
||||
// }
|
||||
// }
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
s1 = s1.replace("法院将于", "").replace(" ", "")
|
||||
def array = s1.split("至")
|
||||
return array
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def getStartPrice(def s) {
|
||||
Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
if (s1.contains("万元")) {
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","")
|
||||
return (int) (s1.toDouble() * 10000)
|
||||
} else {
|
||||
s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","")
|
||||
if (s1.isInteger()) {
|
||||
return s1.toInteger()
|
||||
} else {
|
||||
return (int) (s1.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def getEstateType(def s) {
|
||||
Pattern pattern = Pattern.compile('房屋类型:.*;')
|
||||
Matcher matcher = pattern.matcher(s)
|
||||
|
||||
List<String> list = new ArrayList()
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group())
|
||||
}
|
||||
if (list.size() >= 1) {
|
||||
String s1 = list.get(0)
|
||||
if (s1) {
|
||||
s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";"))
|
||||
return s1.replace(")", "").replace("】", "")
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def findEstateType(def s) {
|
||||
if (s) {
|
||||
def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]
|
||||
for (def type : villTypes) {
|
||||
if (s.contains(type)) {
|
||||
return "别墅"
|
||||
}
|
||||
}
|
||||
|
||||
def districtTypes = ["住宅", "公寓"]
|
||||
for (def type : districtTypes) {
|
||||
if (s.contains(type)) {
|
||||
return "普通住宅"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static def filterChar(def detailMap) {
|
||||
def newDetailMap = [:]
|
||||
detailMap.each { def k, v ->
|
||||
if (k.toString().contains(".")) {
|
||||
newDetailMap.put(k.toString().replace(".", "\\u002e"), v)
|
||||
} else {
|
||||
newDetailMap.put(k, v)
|
||||
}
|
||||
}
|
||||
return newDetailMap
|
||||
}
|
||||
|
||||
/**
|
||||
* 匹配符合条件的字符串
|
||||
*
|
||||
* @param str
|
||||
* @param pattern
|
||||
* @return
|
||||
*/
|
||||
static String match(String str, Pattern pattern) {
|
||||
if (str == null) {
|
||||
return null
|
||||
}
|
||||
Matcher matcher = pattern.matcher(str)
|
||||
if (matcher.find()) {
|
||||
return matcher.group()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* font-secret字符串专用解密工具
|
||||
*
|
||||
* @param key 密匙
|
||||
* @param encodeString 加密后的字符串
|
||||
* @return 解密后的字符串
|
||||
*/
|
||||
static String decodeString(String key, String encodeString) {
|
||||
if (!key) {
|
||||
return ""
|
||||
}
|
||||
try {
|
||||
//base64解码,初始化字体
|
||||
Base64 base64 = new Base64()
|
||||
byte[] ss = base64.decodeBase64(key)
|
||||
InputStream inputStream = new ByteArrayInputStream(ss)
|
||||
Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream)
|
||||
FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false)
|
||||
GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "")
|
||||
|
||||
//获取font中字形的映射关系,字段为private,使用反射
|
||||
Class<?> clazz = Font.class
|
||||
Field[] fs = clazz.getDeclaredFields()
|
||||
Font2DHandle font2DHandle = null
|
||||
for (int i = 0; i < fs.length; i++) {
|
||||
fs[i].setAccessible(true)// 将目标属性设置为可以访问
|
||||
if (fs[i].getName().equals("font2DHandle")) {
|
||||
font2DHandle = (Font2DHandle) fs[i].get(dynamicFont)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//得到映射关系
|
||||
Font2D font2D = font2DHandle.font2D
|
||||
TrueTypeFont trueTypeFont = (TrueTypeFont) font2D
|
||||
TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper()
|
||||
|
||||
//开始解密,encodeString为加密后的字符串
|
||||
StringBuffer buffer = new StringBuffer()
|
||||
char[] chars = encodeString.toCharArray()
|
||||
for (int i = 0; i < chars.length; i++) {
|
||||
buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1)
|
||||
}
|
||||
return buffer.toString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"]
|
||||
|
||||
static String urlToSiteName(String url) {
|
||||
if (StringUtils.isEmpty(url)) {
|
||||
return null
|
||||
}
|
||||
|
||||
for (String suffix : SITE_SUFFIXS) {
|
||||
if (url.indexOf(suffix) > -1) {
|
||||
String s = StringUtils.substringBefore(url, suffix)
|
||||
if (s.indexOf(".") > -1) {
|
||||
s = StringUtils.substringAfterLast(s, ".")
|
||||
if (StringUtils.isNotBlank(s)) {
|
||||
return s + suffix
|
||||
}
|
||||
} else if (s.indexOf("://") > -1) {
|
||||
s = StringUtils.substringAfterLast(s, "://")
|
||||
if (StringUtils.isNotBlank(s)) {
|
||||
return s + suffix
|
||||
}
|
||||
} else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) {
|
||||
return s + suffix
|
||||
}
|
||||
|
||||
return null
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static double getRound(int scale, double dSource) {
|
||||
BigDecimal deSource = new BigDecimal(dSource)
|
||||
return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue()
|
||||
}
|
||||
|
||||
static def filterEmpty(def s){
|
||||
if(s && s.contains("暂无")){
|
||||
return null
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
}
|
||||
+348
@@ -0,0 +1,348 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import javafx.scene.input.DataFormat
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
import org.openqa.selenium.By
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
class IcbcCrawl {
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
static main(args) {
|
||||
def task = new IcbcCrawl();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this"
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
driver.get(url)
|
||||
driver.findElementById("category_SM310000000000000006").click()
|
||||
Thread.sleep(3000)
|
||||
|
||||
|
||||
def urls = getAllDetailLinks(driver)
|
||||
println "共:"+urls.size()
|
||||
|
||||
detailCrawling(urls, driver)
|
||||
|
||||
}
|
||||
|
||||
def getAllDetailLinks(driver){
|
||||
def stop = false
|
||||
def urls = []
|
||||
while (!stop){
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc)
|
||||
if(eles){
|
||||
for(def e : eles){
|
||||
def href = e.select(".li_2_name a").attr("href")
|
||||
urls << "https://gf.trade.icbc.com.cn"+href
|
||||
}
|
||||
def nextPage = JQueryUtils.select("a:contains(下一页)", doc)
|
||||
if(nextPage){
|
||||
driver.findElement(By.linkText("下一页")).click();
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
// if(urls.size()>=10){
|
||||
// stop = true
|
||||
// }
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
def detailCrawling(def urls, driver){
|
||||
def rets = []
|
||||
for(def url : urls){
|
||||
driver.get(url)
|
||||
Thread.sleep(5000)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
def result = parser(doc, url)
|
||||
if (result) {
|
||||
rets << result
|
||||
println rets.size()
|
||||
}
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
|
||||
rets = []
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
saveData(rets)
|
||||
print "完"
|
||||
}
|
||||
}
|
||||
|
||||
def parser(doc, url){
|
||||
def result = [:]
|
||||
if(doc){
|
||||
def title = JQueryUtils.selectText("#prodnameb", doc)
|
||||
def id = url.substring(url.indexOf("="), url.length())
|
||||
if(title){
|
||||
|
||||
def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc)
|
||||
if(cityStr && cityStr.contains("...")){
|
||||
def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc)
|
||||
if(cityEles){
|
||||
cityStr = cityEles[0].attr("title")
|
||||
}
|
||||
}
|
||||
if(cityStr && cityStr.contains("省")){
|
||||
result.province = cityStr.substring(0, cityStr.indexOf("省"))
|
||||
result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length())
|
||||
}else{
|
||||
if(cityStr.contains("自治区")){
|
||||
result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3)
|
||||
result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length())
|
||||
}else{
|
||||
result.city = cityStr.substring(0, cityStr.indexOf("市"))
|
||||
}
|
||||
}
|
||||
|
||||
if(!result.city){
|
||||
return null
|
||||
}
|
||||
result.title = title
|
||||
def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
|
||||
result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","")
|
||||
|
||||
def t1 = JQueryUtils.selectText("#pricep", doc)
|
||||
if(t1 && t1.contains("原因")){
|
||||
result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length())
|
||||
}
|
||||
result.type = JQueryUtils.selectText("dl:contains(类 型:) dd", doc)
|
||||
def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc)
|
||||
if(enrollCount) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText(["#visitorCount"], doc)
|
||||
if(lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText(["#remindPPNum"], doc)
|
||||
if(callCount) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim()
|
||||
if(curPriceStr.isInteger()) {
|
||||
result.curPrice = curPriceStr.toInteger()
|
||||
}else {
|
||||
result.curPrice = (int)(curPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc)
|
||||
if(startPriceEles) {
|
||||
def startPriceStr = startPriceEles[0].attr("title")
|
||||
if(startPriceStr){
|
||||
startPriceStr = startPriceStr.replace(",", "").replace("¥","")
|
||||
|
||||
if(startPriceStr.isInteger()) {
|
||||
result.startPrice = startPriceStr.toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(startPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
|
||||
if(addPriceEles){
|
||||
def addPriceStr = addPriceEles[0].attr("title")
|
||||
if(addPriceStr){
|
||||
addPriceStr = addPriceStr.replace(",", "").replace("¥","")
|
||||
if(addPriceStr) {
|
||||
result.addPrice = addPriceStr.toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc)
|
||||
if(evaluationPriceEles){
|
||||
def evaluationPriceStr = evaluationPriceEles[0].attr("title")
|
||||
if(evaluationPriceStr){
|
||||
evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","")
|
||||
if(evaluationPriceStr.isInteger()) {
|
||||
result.evaluationPrice = evaluationPriceStr.toDouble().toInteger()
|
||||
}else {
|
||||
result.evaluationPrice = (int)(evaluationPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc)
|
||||
if(depositPriceEles){
|
||||
def v = depositPriceEles[0].attr("title")
|
||||
if(v){
|
||||
v = v.replace(",", "").replace("¥","")
|
||||
if(v.isInteger()) {
|
||||
result.depositPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.depositPrice = (int)(v.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc)
|
||||
result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc)
|
||||
result.court = JQueryUtils.selectText("#goToCourt", doc)
|
||||
def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc)
|
||||
if(mobileEles){
|
||||
result.mobile = mobileEles[0].attr("title")
|
||||
}
|
||||
|
||||
def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("法院将于")
|
||||
def end = s1.indexOf("止")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = PaimaiDataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时")
|
||||
result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时")
|
||||
}
|
||||
}
|
||||
|
||||
def trs = JQueryUtils.select([".rygtable table tr"], doc)
|
||||
if(trs) {
|
||||
def detailMap = [:]
|
||||
for(def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if(d_tds && d_tds.size()>=2) {
|
||||
if(d_tds.size()==3) {
|
||||
if(d_tds[2].text() && d_tds[2].text()!="/"){
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
}
|
||||
|
||||
}else {
|
||||
if(d_tds[1].text() && d_tds[1].text()!="/") {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(detailMap) {
|
||||
result.detailMap = DataUtil.filterChar(detailMap)
|
||||
}
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("标的名称")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def imgs =JQueryUtils.select("#scroll li img", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
for(def img : imgs) {
|
||||
def href = img.attr("src")
|
||||
if(href.contains("icbc.com")) {
|
||||
imgLinks << "https:"+href
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
|
||||
def detailStr = JQueryUtils.selectText(".detail_list", doc)
|
||||
if(detailStr){
|
||||
def area = DataUtil.getAreaNumber(detailStr, 5000)
|
||||
if(area) {
|
||||
result.area = area.toDouble()
|
||||
if(result.tradePrice) {
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
result.introduce = detailStr.toString()
|
||||
result.url = url
|
||||
result.mDate = new Date()
|
||||
result.source = "融e够"
|
||||
return result;
|
||||
}
|
||||
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getCallCount(id){
|
||||
def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id
|
||||
def s = ABuYunUtil.doPost(url, null, null, "utf-8", true)
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
println jsonObject
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = paimaiResultColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimaiResultColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return add
|
||||
}
|
||||
}
|
||||
+540
@@ -0,0 +1,540 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.OpenApiUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.openqa.selenium.WebDriver
|
||||
import org.openqa.selenium.chrome.ChromeDriver
|
||||
|
||||
import javax.sound.midi.MidiSystem
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class JdPaimaiCrawl {
|
||||
|
||||
DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai")
|
||||
|
||||
static main(args) {
|
||||
def task = new JdPaimaiCrawl();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def existsIds = []//findAllIds()
|
||||
println "已存在:"+existsIds.size()
|
||||
System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe");
|
||||
System.setProperty("http.nonProxyHosts", "localhost");
|
||||
WebDriver driver = new ChromeDriver()
|
||||
def links = getList(driver)
|
||||
def ids = getDetailLinks(links, driver)
|
||||
println "共:"+ids.size()
|
||||
parserList( existsIds,ids, driver)
|
||||
|
||||
}
|
||||
|
||||
def getList(driver){
|
||||
def list = []
|
||||
list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268"
|
||||
// for(def i=1;i<=32;i++){
|
||||
// if(i==8){
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268"
|
||||
// }else if(i==5){
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268"
|
||||
// }else if(i==17){
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268"
|
||||
// }else if(i==19){
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268"
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268"
|
||||
// }else{
|
||||
// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268"
|
||||
// }
|
||||
|
||||
// }
|
||||
def links = []
|
||||
for(def url : list){
|
||||
driver.get(url)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
//def document = ABuYunUtil.doPost(url, null, null, "utf-8", true)
|
||||
if(doc){
|
||||
def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")"))
|
||||
JSONObject obj = JSONObject.fromObject(json);
|
||||
if(obj && obj.total){
|
||||
def total = obj.total
|
||||
def page = total/40
|
||||
def p = page.toInteger()
|
||||
if(page > p){
|
||||
p = p +1
|
||||
}
|
||||
|
||||
for(def i=1;i<=p;i++){
|
||||
links << url+"&page="+i
|
||||
}
|
||||
// if(links){
|
||||
// break;
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return links;
|
||||
}
|
||||
|
||||
def getDetailLinks(links, driver){
|
||||
def rets = []
|
||||
def citys = ["北京","上海","天津","重庆"]
|
||||
for(def link : links){
|
||||
driver.get(link)
|
||||
def doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc) {
|
||||
def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")"))
|
||||
JSONObject obj = JSONObject.fromObject(json);
|
||||
if (obj && obj.ls) {
|
||||
for(def o : obj.ls){
|
||||
def map = [:]
|
||||
if(citys.contains(o.province)){
|
||||
map.province = o.province
|
||||
map.city = o.province
|
||||
map.region = o.city
|
||||
}else{
|
||||
map.province = o.province
|
||||
map.city = o.city
|
||||
}
|
||||
map.id = o.id.toString()
|
||||
|
||||
rets << map
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def parserList(existsIds,list, driver){
|
||||
def rets = []
|
||||
for(def map : list) {
|
||||
if(!existsIds.contains(map.id)){
|
||||
def url = "https://paimai.jd.com/" + map.id
|
||||
def doc = getDocument(url, driver)
|
||||
if (doc) {
|
||||
def result = parsering(doc, url, map.province, map.city, map.region)
|
||||
if (result) {
|
||||
rets << result
|
||||
}
|
||||
if (rets.size() % 100 == 0) {
|
||||
def add= saveData(rets)
|
||||
println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add
|
||||
rets = []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(rets.size()>0){
|
||||
saveData(rets)
|
||||
print "完"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def saveData(def results){
|
||||
def add = 0
|
||||
for(def ret : results){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", ret.url)
|
||||
def dbo = paimaiResultColl.findOne(q);
|
||||
|
||||
DBObject obj = new BasicDBObject();
|
||||
ret.each{k,v->
|
||||
obj.put(k,v)
|
||||
}
|
||||
|
||||
if(dbo){
|
||||
obj.m_date = new Date()
|
||||
paimaiResultColl.update(q, new BasicDBObject('$set', obj))
|
||||
}else{
|
||||
add++
|
||||
obj.c_date = new Date()
|
||||
obj.m_date = new Date()
|
||||
try{
|
||||
paimaiResultColl.save(obj)
|
||||
}catch(Exception e){
|
||||
println obj.url
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return add
|
||||
}
|
||||
|
||||
def parsering(doc, def url, def province, def city, def region){
|
||||
def result = [:]
|
||||
if(doc) {
|
||||
result.proince = province
|
||||
result.city = city
|
||||
if(region){
|
||||
result.region = region
|
||||
}
|
||||
def title = JQueryUtils.selectText([".pm-name"], doc)
|
||||
result.title = title
|
||||
result.address = JQueryUtils.selectText([".pm-location em"], doc)
|
||||
if(result.address) {
|
||||
//调用获取城市
|
||||
// if(!city) {
|
||||
// city = RegionData.formatCity(cityMap.name)
|
||||
// }
|
||||
|
||||
if(city) {
|
||||
def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true)
|
||||
if (rMap) {
|
||||
result.matchCity = rMap.city
|
||||
result.matchRegion = rMap.region
|
||||
result.matchName = rMap.name
|
||||
result.matchAddress = rMap.address
|
||||
result.houseNum = rMap.house_num
|
||||
result.roomNum = rMap.room_num
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
result.status = JQueryUtils.selectText([".pm-summary .state"], doc)
|
||||
|
||||
def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc)
|
||||
if(enrollCount) {
|
||||
result.enrollCount = enrollCount.toInteger()
|
||||
}
|
||||
|
||||
def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc)
|
||||
if(lookCount) {
|
||||
result.lookCount = lookCount.toInteger()
|
||||
}
|
||||
|
||||
def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc)
|
||||
if(callCount) {
|
||||
result.callCount = callCount.toInteger()
|
||||
}
|
||||
|
||||
|
||||
def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc)
|
||||
if(curPriceStr) {
|
||||
curPriceStr = curPriceStr.replace(",", "")
|
||||
if(curPriceStr.isInteger()) {
|
||||
result.curPrice = curPriceStr.toInteger()
|
||||
}else {
|
||||
result.curPrice = (int)(curPriceStr.toDouble())
|
||||
}
|
||||
}
|
||||
|
||||
def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc)
|
||||
if(startPriceStr) {
|
||||
startPriceStr = startPriceStr.replace(",", "")
|
||||
|
||||
if(startPriceStr.isInteger()) {
|
||||
result.startPrice = startPriceStr.toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(startPriceStr.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc)
|
||||
if(tradePriceStr) {
|
||||
tradePriceStr = tradePriceStr.replace(",", "")
|
||||
if(tradePriceStr.isInteger()) {
|
||||
result.tradePrice = tradePriceStr.toInteger()
|
||||
}else {
|
||||
result.tradePrice = (int)(tradePriceStr.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def finishDateStr = JQueryUtils.selectText([".endtime"], doc)
|
||||
if(finishDateStr && finishDateStr.contains("结束时间")) {
|
||||
finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" "))
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss")
|
||||
}
|
||||
|
||||
def lis = JQueryUtils.select(".description ul li", doc)
|
||||
if(lis && lis.size()) {
|
||||
for(def li : lis) {
|
||||
def str = li.select("span").text()
|
||||
if(str.contains("起拍")) {
|
||||
def v= li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
if(v.isInteger()) {
|
||||
result.startPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.startPrice = (int)(v.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
}else if(str.contains("评估价")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v.isInteger()) {
|
||||
result.evaluationPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.evaluationPrice = (int)(v.toDouble())
|
||||
}
|
||||
}else if(str.contains("加价幅度")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
result.addPrice = v.toDouble().toInteger()
|
||||
}
|
||||
}else if(str.contains("保证金")) {
|
||||
def v = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
if(v) {
|
||||
if(v.isInteger()) {
|
||||
result.depositPrice = v.toDouble().toInteger()
|
||||
}else {
|
||||
result.depositPrice = (int)(v.toDouble())
|
||||
}
|
||||
|
||||
}
|
||||
}else if(str.contains("延时周期")) {
|
||||
|
||||
result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
|
||||
}else if(str.contains("优先竞买权人")) {
|
||||
result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc)
|
||||
if(trs) {
|
||||
def detailMap = [:]
|
||||
for(def tr : trs) {
|
||||
def d_tds = tr.select("td")
|
||||
if(d_tds && d_tds.size()>=2) {
|
||||
if(d_tds.size()==3) {
|
||||
detailMap.put(d_tds[1].text(), d_tds[2].text())
|
||||
}else {
|
||||
detailMap.put(d_tds[0].text(), d_tds[1].text())
|
||||
}
|
||||
}else {
|
||||
if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) {
|
||||
detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", ""))
|
||||
}
|
||||
}
|
||||
}
|
||||
if(detailMap) {
|
||||
result.detailMap = DataUtil.filterChar(detailMap)
|
||||
}
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("拍品名称")
|
||||
if(!result.address) {
|
||||
result.address = detailMap.get("标的名称")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc)
|
||||
if(imgs) {
|
||||
def imgLinks = []
|
||||
for(def img : imgs) {
|
||||
def href = img.attr("href")
|
||||
if(href.contains("http")) {
|
||||
imgLinks << href
|
||||
}else {
|
||||
imgLinks << url.substring(0, url.indexOf(":")+1)+href
|
||||
}
|
||||
}
|
||||
result.imgUrls = imgLinks
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
//解析面积
|
||||
def article = JQueryUtils.selectText([".floor-content"], doc)
|
||||
if(article) {
|
||||
def area = DataUtil.getAreaNumber(article, 5000)
|
||||
if(!area) {
|
||||
article = JQueryUtils.selectText(["body"], doc)
|
||||
if(article) {
|
||||
area = DataUtil.getAreaNumber(article, 5000)
|
||||
}
|
||||
}
|
||||
|
||||
if(area) {
|
||||
result.area = area.toDouble()
|
||||
if(result.tradePrice) {
|
||||
result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc)
|
||||
if(s1) {
|
||||
def start = s1.indexOf("将于")
|
||||
def end = s1.indexOf("止")
|
||||
if(start>-1 && end>-1 && end>start){
|
||||
s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止"))
|
||||
}
|
||||
def dateRangs = DataUtil.getDateRange(s1)
|
||||
if(dateRangs && dateRangs.size()==2) {
|
||||
|
||||
for(def j=0;j<dateRangs.size();j++) {
|
||||
def date = dateRangs[j]
|
||||
def pattern = null
|
||||
if(date.endsWith("日时")||date.endsWith("日")) {
|
||||
pattern = "yyyy年MM月dd日"
|
||||
}else {
|
||||
|
||||
def ind = date.indexOf("日")
|
||||
if(date.indexOf("时")==-1 && date.toString().length()>ind+1) {
|
||||
date = date +"时"
|
||||
}
|
||||
|
||||
pattern = "yyyy年MM月dd日HH时"
|
||||
}
|
||||
if(j==0) {
|
||||
try{
|
||||
result.startDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
}else {
|
||||
if(!result.finishDate) {
|
||||
result.finishDate = DateUtils.parseFromTightByPattern(date, pattern)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!result.startPrice) {
|
||||
result.startPrice = DataUtil.getStartPrice(s1)
|
||||
}
|
||||
|
||||
def estateType = DataUtil.findEstateType(s1)
|
||||
if(estateType) {
|
||||
result.matchEstateType = estateType
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!result.startDate){
|
||||
def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","")
|
||||
result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日")
|
||||
}
|
||||
if(result.finishDate){
|
||||
result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01")
|
||||
}
|
||||
if(result.startDate){
|
||||
result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01")
|
||||
}
|
||||
|
||||
result.content = com.fangjia.core.util.DataUtil.zip(doc.toString())
|
||||
result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString()
|
||||
result.url = url
|
||||
result.city = city
|
||||
result.mDate = new Date()
|
||||
result.source = "京东"
|
||||
return result;
|
||||
}
|
||||
|
||||
def getDocument(url, driver){
|
||||
def stop = false
|
||||
def error = 0
|
||||
def doc = null
|
||||
while(!stop){
|
||||
try{
|
||||
driver.get(url)
|
||||
doc = Jsoup.parse(driver.getPageSource());
|
||||
if(doc){
|
||||
stop = true;
|
||||
}else{
|
||||
error++
|
||||
Thread.sleep(5000)
|
||||
}
|
||||
}catch(Exception e){
|
||||
error++
|
||||
}
|
||||
if(error>=10){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
return doc
|
||||
}
|
||||
def findAllIds(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("url", Pattern.compile(".*jd.com.*"))
|
||||
def dbcursor = paimaiResultColl.find(q)
|
||||
def ids = []
|
||||
for(def d : dbcursor){
|
||||
def url = d.url
|
||||
if(url){
|
||||
def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length())
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
return ids;
|
||||
}
|
||||
|
||||
}
|
||||
+34
-8
@@ -37,7 +37,27 @@ public class LianjiaCrawlByChrome {
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.standardRegion()
|
||||
task.standardRegion()
|
||||
// task.updadteDealPriceField()
|
||||
}
|
||||
|
||||
def updadteDealPriceField(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("hall", "0")
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
println "[${index}] "+city +" "+count
|
||||
if(count && count>0){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("hall", 0)
|
||||
DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true);
|
||||
println "[ÐÞ¸Ä]"+city +" "+ count
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run1(){
|
||||
@@ -187,7 +207,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
}else{
|
||||
@@ -218,7 +238,7 @@ public class LianjiaCrawlByChrome {
|
||||
def durl = e.select(".title a").attr("href")
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
parse(city, e)
|
||||
@@ -227,6 +247,10 @@ public class LianjiaCrawlByChrome {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
page++;
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -254,7 +278,7 @@ public class LianjiaCrawlByChrome {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
}
|
||||
if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) {
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -374,8 +398,8 @@ public class LianjiaCrawlByChrome {
|
||||
def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "")
|
||||
if (roomStr) {
|
||||
detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger()
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü"))
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ"))
|
||||
detail.hall = roomStr.substring(roomStr.indexOf("ÊÒ") + 1, roomStr.indexOf("Ìü")).toInteger()
|
||||
detail.toilet = roomStr.substring(roomStr.indexOf("ÎÀ") - 1, roomStr.indexOf("ÎÀ")).toInteger()
|
||||
}
|
||||
detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "")
|
||||
detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "")
|
||||
@@ -416,12 +440,14 @@ public class LianjiaCrawlByChrome {
|
||||
def cityDoc = Jsoup.parse(driver.getPageSource());
|
||||
def cityEles = JQueryUtils.select(".city_province a", cityDoc)
|
||||
def cityMap = [:]
|
||||
def citys= ["¸£ÖÝ"]
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
// if(!href.contains("fang.lianjia") && e.text().contains("¸£ÖÝ")){
|
||||
if (!href.contains("fang.lianjia")){
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user