diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java index 4f59fe761..354e4c4ff 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java @@ -13,7 +13,7 @@ import java.util.*; */ public class BaiDuApiUtils { - private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5"; + private static String AK = "ajgWIBd9rYsXed1hcDQqYvkX"; public static String[] AKS = new String[]{"hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg", "4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784", "9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224", diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index 93218b265..d4cc2d5da 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -25,6 +25,7 @@ import java.util.zip.GZIPOutputStream; import com.hankcs.hanlp.HanLP; import com.hankcs.hanlp.dictionary.py.Pinyin; +import com.sun.javafx.scene.transform.TransformUtils; import net.sf.json.JSON; import net.sf.json.JSONArray; import net.sf.json.JSONNull; @@ -1435,6 +1436,25 @@ public class DataUtil { return null; } + /** + * 匹配符合条件的字符串 + * + * @param str + * @param pattern + * @return + */ + public static List matchAll(String str, Pattern pattern) { + if (str == null) { + return null; + } + List list = new ArrayList(); + Matcher matcher = pattern.matcher(str); + while (matcher.find()) { + list.add(matcher.group()); + } + return list; + } + public static boolean isNumeric(String str) { try { Double.parseDouble(str); @@ -1805,11 +1825,26 @@ public class DataUtil { } } - public static void main(String[] args) { - System.out.println(getFirstChar("文裕花园")); - System.out.println(getFirstChar("文怡花园")); + public static boolean isChineseChar(String c) { + if (StringUtils.isNotBlank(c)) { + String temp = String.valueOf(c); + Pattern pattern = Pattern.compile("[\\u4e00-\\u9fa5\\w]+"); + Matcher matcher = pattern.matcher(temp); + return matcher.matches(); + } + return false; + } - System.out.println(FourUpSixInto("258.393", 2)); + + + + + + public static void main(String[] args) { + System.out.println(isChineseChar("6文怡a花园")); +// System.out.println(getFirstChar("文怡花园")); +// +// System.out.println(FourUpSixInto("258.393", 2)); } diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/ExcelUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/ExcelUtil.java index a24864225..509490d97 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/ExcelUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/ExcelUtil.java @@ -58,7 +58,7 @@ public class ExcelUtil { row.setHeightInPoints(35); Font font = workBook.createFont(); font.setFontName("宋体"); - font.setBoldweight(Font.BOLDWEIGHT_BOLD); + // font.setBoldweight(Font.BOLDWEIGHT_BOLD); font.setFontHeightInPoints((short) 12); CellStyle cellStyle = workBook.createCellStyle(); cellStyle.setFont(font); diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java index 25f4d5a12..234c29d79 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java @@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils { public static void main(String[] args) { for (int i = 0; i < 1000; i++) { - System.out.println(random(0, 10)); + System.out.println(random(-9, 9)); } } } diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/SeaweedUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/SeaweedUtils.java new file mode 100644 index 000000000..698833bd5 --- /dev/null +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/SeaweedUtils.java @@ -0,0 +1,202 @@ +package com.fangjia.core.util; + +import org.apache.commons.lang.StringUtils; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public class SeaweedUtils { + public static void main(String[] args){ + System.out.println(formatVirescencePercent("1")); + } + /** + * 标准化输出绿化率 + */ + public static String formatVirescencePercent(String virescence_percent) { + // 绿化率 + if (StringUtils.isNotBlank(virescence_percent) && checkValue(virescence_percent)!=null) { + double val = Double.valueOf(virescence_percent); + if (val < 1) { + return String.valueOf(val * 100); + } + if (val == 1) { + return String.valueOf(val * 10); + } + if (val >= 70) { + return "65"; + } + return String.valueOf(val); + } + return null; + } + + /** + * 标准化容积率输出 + */ + public static String formatCubagePercent(String cubage_percent) { + if (StringUtils.isNotBlank(cubage_percent) && checkValue(cubage_percent)!=null) { + double val = Double.valueOf(cubage_percent); + if (val >= 7 && val < 8) { + return "6.2"; + } + if (val >= 8 && val < 9) { + return "6.3"; + } + if (val >= 9 && val < 10) { + return "6.5"; + } + if (val >= 10) { + return "6.7"; + } + return String.valueOf(val); + } + return null; + } + + /** + * 标准化车位输出 + */ + public static String formatCarSiteCount(String car_site_count) { + if (StringUtils.isNotBlank(car_site_count) && checkValue(car_site_count)!=null) { + long val = Long.valueOf(car_site_count); + if (val > 2500 || val < 100) { + return ""; + } + return String.valueOf(val); + } + return null; + } + + /** + * 标准化户数输出 + */ + public static String formatHouseholdCount(String household_count) { + if (StringUtils.isNotBlank(household_count) && checkValue(household_count)!=null ) { + if (Integer.valueOf(household_count) < 50) { + return null; // 户数 + } + return household_count; + } + return null; + } + + /** + * 标准化总占地面积输出 + */ + public static String formatAllArea(String all_area) { + if (StringUtils.isNotBlank(all_area)) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + String val = all_area.toString(); + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000); + } else if(checkValue(all_area)!=null) { + formattedVal = Double.valueOf(all_area); + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return null; + } + return String.valueOf(formattedVal); + } + return null; + } + + /** + * 提取浮点数与整数 + */ + static String getNumber(String input) { + Pattern pattern = Pattern.compile("\\d+(\\.\\d+)?"); + Matcher matcher = pattern.matcher(input); + List list = new ArrayList(); + while (matcher.find()) { + list.add(matcher.group()); + } + if(list.size()>0){ + return list.get(0); + } + return null; + } + + + + + static Float checkValue(String value){ + if(StringUtils.isNotBlank(value)){ + value = getNumber(value); + if(StringUtils.isNotBlank(value)){ + value = subZeroAndDot(value); + if(isNumber(value)){ + Float v = Float.parseFloat(value); + if(v>0){ + return v; + }else{ + return null; + } + }else if(isFloat(value)){ + Float v = Float.parseFloat(value); + if(v>0){ + return v; + }else{ + return null; + } + } else{ + return null; + } + } + } + return null; + } + + static boolean isNumber(String str){ + try { + Integer.parseInt(str); + } catch (NumberFormatException e) { + return false; + } + return true; + } + + static boolean isFloat(String str){ + try { + Float.parseFloat(str); + } catch (NumberFormatException e) { + return false; + } + return true; + } + + /** + * 正则表达式去掉多余的.与0 + * @param + * @return + */ + static String subZeroAndDot(String v){ + + if(StringUtils.isNotBlank(v)){ + if(v.indexOf(".")!=-1){ + Double myNum2 = Double.valueOf(v); + BigDecimal b = new java.math.BigDecimal(myNum2); + Double myNum3 = b.setScale(2, java.math.BigDecimal.ROUND_HALF_UP).doubleValue(); + v = myNum3.toString(); + } + + if(v.indexOf(".")>-1){ + while(true) + { + if(v.endsWith("0")){ + v = v.substring(0,v.length()-1); + }else{ + break; + } + } + if(v.endsWith(".")){ + v=v.substring(0,v.length()-1); + } + } + } + return v; + } + +} diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SaleLeadsController.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SaleLeadsController.java index 3f6881dd5..9596069f4 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SaleLeadsController.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/SaleLeadsController.java @@ -1,5 +1,9 @@ package com.fangjia.coresite.controllers; +import java.io.UnsupportedEncodingException; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.net.URLEncoder; import java.util.Date; import java.util.List; @@ -7,6 +11,7 @@ import java.util.List; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import com.fangjia.core.util.DataUtil; import org.apache.commons.lang.StringUtils; import com.fangjia.coresite.model.JsonModel; @@ -37,7 +42,7 @@ public class SaleLeadsController { } public Object doAddAction(SaleLeadsModel model, HttpServletRequest req, - HttpServletResponse res) { + HttpServletResponse res) throws UnsupportedEncodingException, NoSuchMethodException, IllegalAccessException, InvocationTargetException { model.setViewType("redirect"); String refUrl = model.getBackUrl(); @@ -122,6 +127,8 @@ public class SaleLeadsController { } } + + saleLeads = converValue(saleLeads); saleLeadsService.save(saleLeads); if(StringUtils.isNotBlank(saleLeads.getBusiness()) && saleLeads.getBusiness().contains("_")){ @@ -162,6 +169,30 @@ public class SaleLeadsController { model.setViewParam("location", url); return model; } + + private SaleLeads converValue(SaleLeads saleLeads) throws NoSuchMethodException, InvocationTargetException, IllegalAccessException, UnsupportedEncodingException { +// + Field[] fields = saleLeads.getClass().getDeclaredFields(); + for(Field s : fields){ + String name = s.getName(); + String type = s.getGenericType().toString(); + if(type.contains("java.lang.String")){ + String fildName = name.substring(0,1).toUpperCase()+name.substring(1); + Method m = saleLeads.getClass().getMethod("get"+fildName); + String value = (String)m.invoke(saleLeads); + if(StringUtils.isNotBlank(value)){ + String s1 = value.replaceAll("\\pP|\\pS", ""); + if(!DataUtil.isChineseChar(s1)){ + String newValue = new String(value.getBytes("ISO8859-1"), "UTF-8"); + m = saleLeads.getClass().getMethod("set"+fildName,String.class); + m.invoke(saleLeads, newValue); + } + } + } + } + return saleLeads; + + } public Object addDownInfoAction(SaleLeadsModel model, HttpServletRequest req, diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/TrendController.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/TrendController.java index c3a3756a9..cccf755b8 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/TrendController.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/TrendController.java @@ -188,6 +188,8 @@ public class TrendController { // avgPrice = model.getRegionStatBo().getAvgPrice(); // } + int count = districtService.findCityDistrictCount(city); + model.setCount(count); model.setTrendView(trendView); //model.setLastestHouse(getRelatedHouse(model.getRegion(), avgPrice)); return model; diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/model/TrendModel.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/model/TrendModel.java index 6e7cfda75..a06cb6a31 100644 --- a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/model/TrendModel.java +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/model/TrendModel.java @@ -53,6 +53,7 @@ public class TrendModel extends BaseModel implements TrendViewModel { private int showBlock; // 小区走势图里显示板块线,不为0时显示 private String errMsg; //错误信息 private int code; //错误code 1成功 -1失败 + private int count; public String getKeyword() { return keyword; } @@ -317,4 +318,11 @@ public class TrendModel extends BaseModel implements TrendViewModel { this.code = code; } + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } } diff --git a/projects/fangjia-coresite/src/main/webapp/company/about.ftl b/projects/fangjia-coresite/src/main/webapp/company/about.ftl index 778557075..16b30ed80 100644 --- a/projects/fangjia-coresite/src/main/webapp/company/about.ftl +++ b/projects/fangjia-coresite/src/main/webapp/company/about.ftl @@ -158,37 +158,36 @@ <#-- --> - + - + - + - + - + - diff --git a/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl b/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl index c4fb35940..ea155f653 100644 --- a/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl +++ b/projects/fangjia-coresite/src/main/webapp/trend/trend.ftl @@ -125,6 +125,7 @@
<#if districtStatBo?? || regionStatBo??> + <#if count??>
${count!}
<#if districtStatBo??>
<#if districtStatBo.lastTimeDistrictStat??>${districtStatBo.lastTimeDistrictStat.avgPrice}<#else>鏆傛棤
<#if districtStatBo.districtStat??>${districtStatBo.districtStat.avgPrice}<#else>鏆傛棤
diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java new file mode 100644 index 000000000..5f895886b --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDao.java @@ -0,0 +1,7 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.BsEvaluation; + +public interface BsEvaluationDao extends MongoDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java new file mode 100644 index 000000000..e4d7fc080 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/BsEvaluationDaoImpl.java @@ -0,0 +1,8 @@ +package com.fangjia.dal.mongo.dao; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.model.BsEvaluation; + +public class BsEvaluationDaoImpl extends MongoDaoImpl implements + BsEvaluationDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java index fa5f7cb2f..e7a406dc8 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/fangjia/model/PaiMai.java @@ -99,6 +99,9 @@ public class PaiMai implements MongoEntity { private Integer marketAvgPrice; /**物业类型*/ private String matchEstateType; + private String houseNum; + private String unitNum; + private String roomNum; @Id @GeneratedValue @@ -488,4 +491,28 @@ public class PaiMai implements MongoEntity { public void setMatchEstateType(String matchEstateType) { this.matchEstateType = matchEstateType; } + + public String getHouseNum() { + return houseNum; + } + + public void setHouseNum(String houseNum) { + this.houseNum = houseNum; + } + + public String getUnitNum() { + return unitNum; + } + + public void setUnitNum(String unitNum) { + this.unitNum = unitNum; + } + + public String getRoomNum() { + return roomNum; + } + + public void setRoomNum(String roomNum) { + this.roomNum = roomNum; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java new file mode 100644 index 000000000..d3b1b28f0 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/BsEvaluation.java @@ -0,0 +1,184 @@ +package com.fangjia.dal.mongo.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; +import java.util.Date; + +@Entity +@Table(name = "business.evaluation") +public class BsEvaluation implements MongoEntity { + private String id; + private String fullAddress; + private String commid; + private String cityName; + private String distName; + private String commName; + private String address; + private Double buildArea; + private Integer floor; + private Integer floorcnt; + private Integer roomcnt; + private Integer livingroomcnt; + private String houseaspect;//南 + private String fitment; + private Date cDate; + private Integer unitPrice; + private Integer totalPrice; + + @Id + @GeneratedValue(strategy = GenerationType.AUTO) + @Column(name = "_id") + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "full_address") + public String getFullAddress() { + return fullAddress; + } + + public void setFullAddress(String fullAddress) { + this.fullAddress = fullAddress; + } + + @Column(name = "commid") + public String getCommid() { + return commid; + } + + public void setCommid(String commid) { + this.commid = commid; + } + + @Column(name = "cityname") + public String getCityName() { + return cityName; + } + + public void setCityName(String cityName) { + this.cityName = cityName; + } + + @Column(name = "distname") + public String getDistName() { + return distName; + } + + public void setDistName(String distName) { + this.distName = distName; + } + + @Column(name = "commname") + public String getCommName() { + return commName; + } + + public void setCommName(String commName) { + this.commName = commName; + } + + @Column(name = "address") + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + @Column(name = "buildarea") + public Double getBuildArea() { + return buildArea; + } + + public void setBuildArea(Double buildArea) { + this.buildArea = buildArea; + } + + @Column(name = "floor") + public Integer getFloor() { + return floor; + } + + public void setFloor(Integer floor) { + this.floor = floor; + } + + @Column(name = "floorcnt") + public Integer getFloorcnt() { + return floorcnt; + } + + public void setFloorcnt(Integer floorcnt) { + this.floorcnt = floorcnt; + } + + @Column(name = "roomcnt") + public Integer getRoomcnt() { + return roomcnt; + } + + public void setRoomcnt(Integer roomcnt) { + this.roomcnt = roomcnt; + } + + @Column(name = "livingroomcnt") + public Integer getLivingroomcnt() { + return livingroomcnt; + } + + public void setLivingroomcnt(Integer livingroomcnt) { + this.livingroomcnt = livingroomcnt; + } + + @Column(name = "houseaspect") + public String getHouseaspect() { + return houseaspect; + } + + public void setHouseaspect(String houseaspect) { + this.houseaspect = houseaspect; + } + + @Column(name = "fitment") + public String getFitment() { + return fitment; + } + + public void setFitment(String fitment) { + this.fitment = fitment; + } + + @Column(name = "c_date") + public Date getcDate() { + return cDate; + } + + public void setcDate(Date cDate) { + this.cDate = cDate; + } + + @Column(name = "unitprice") + public Integer getUnitPrice() { + return unitPrice; + } + + public void setUnitPrice(Integer unitPrice) { + this.unitPrice = unitPrice; + } + + @Column(name = "totalprice") + public Integer getTotalPrice() { + return totalPrice; + } + + public void setTotalPrice(Integer totalPrice) { + this.totalPrice = totalPrice; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java index 36021519c..cc946257b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/DistrictFacilitys.java @@ -63,6 +63,7 @@ public class DistrictFacilitys implements MongoEntity { private String imgUrl3D; private List school; + private List keySchool; @Id @GeneratedValue @@ -380,4 +381,12 @@ public class DistrictFacilitys implements MongoEntity { this.school = school; } + @Column(name = "key_school") + public List getKeySchool() { + return keySchool; + } + + public void setKeySchool(List keySchool) { + this.keySchool = keySchool; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java index a4ababe8e..a46cd7de0 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/DealPrice.java @@ -48,6 +48,11 @@ public class DealPrice implements MongoEntity { private Integer floor;//当前楼层 private Long score; + private String matchId; + private String matchCity; + private String matchRegion; + private String matchName; + @Id @GeneratedValue @Column(name = "_id", unique = true, nullable = false) @@ -296,4 +301,40 @@ public class DealPrice implements MongoEntity { } return true; } + + @Column(name = "match_id") + public String getMatchId() { + return matchId; + } + + public void setMatchId(String matchId) { + this.matchId = matchId; + } + + @Column(name = "match_city") + public String getMatchCity() { + return matchCity; + } + + public void setMatchCity(String matchCity) { + this.matchCity = matchCity; + } + + @Column(name = "match_region") + public String getMatchRegion() { + return matchRegion; + } + + public void setMatchRegion(String matchRegion) { + this.matchRegion = matchRegion; + } + + @Column(name = "match_name") + public String getMatchName() { + return matchName; + } + + public void setMatchName(String matchName) { + this.matchName = matchName; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java index cf98a3665..d41b400ee 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/HouseLoudong.java @@ -448,6 +448,7 @@ public class HouseLoudong extends com.fangjia.dal.mongo.model.HouseLoudong{ this.openTotalPrice = openTotalPrice; } + @Column(name = "district_id") public String getDistrictId() { return districtId; } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java index 712b5fc7d..12cb8ca9b 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/Loudong.java @@ -101,7 +101,11 @@ public class Loudong implements MongoEntity { * seaweed中的小区名 */ private String nameInSeaweed; - + /** + * seaweed中的小区id + */ + private String idInSeaweed; + @Column(name = "remark") public String getRemark() { return remark; @@ -402,4 +406,13 @@ public class Loudong implements MongoEntity { public void setNameInSeaweed(String nameInSeaweed) { this.nameInSeaweed = nameInSeaweed; } + + @Column(name = "id_in_seaweed") + public String getIdInSeaweed() { + return idInSeaweed; + } + + public void setIdInSeaweed(String idInSeaweed) { + this.idInSeaweed = idInSeaweed; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java new file mode 100644 index 000000000..8275f68f8 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityBasePriceDao.java @@ -0,0 +1,9 @@ +package com.fangjia.dal.mongo.open.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityBasePriceDao extends MongoDao { + +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java new file mode 100644 index 000000000..97f9d48de --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/CommunityPoiDao.java @@ -0,0 +1,9 @@ +package com.fangjia.dal.mongo.open.dao; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.CountDistrictStatBycity; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityPoiDao extends MongoDao { + +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java new file mode 100644 index 000000000..c89477b09 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityBasePriceDaoImpl.java @@ -0,0 +1,10 @@ +package com.fangjia.dal.mongo.open.dao.impl; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public class CommunityBasePriceDaoImpl extends MongoDaoImpl implements CommunityBasePriceDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java new file mode 100644 index 000000000..e8a317b04 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/dao/impl/CommunityPoiDaoImpl.java @@ -0,0 +1,10 @@ +package com.fangjia.dal.mongo.open.dao.impl; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.open.dao.ApiVisitCountDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.dal.mongo.tool.model.ApiVisitCount; + +public class CommunityPoiDaoImpl extends MongoDaoImpl implements CommunityPoiDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java new file mode 100644 index 000000000..9321bd542 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityBasePrice.java @@ -0,0 +1,74 @@ +package com.fangjia.dal.mongo.open.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.Table; + +/** + * 小区建行置信度信息 + * + * @author shenjian@fangjia.com + */ +@Entity +@Table(name = "business.community_base_price_gj") +public class CommunityBasePrice implements MongoEntity { + + private static final long serialVersionUID = -8010010773207622878L; + + private String id; + private String city; + private String region; + private String name; + private Double credit; + + @Id + @Column(name = "_id") + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "credit") + public Double getCredit() { + return credit; + } + + public void setCredit(Double credit) { + this.credit = credit; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java new file mode 100644 index 000000000..8427b5aa7 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/open/model/CommunityPoi.java @@ -0,0 +1,135 @@ +package com.fangjia.dal.mongo.open.model; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.*; +import java.util.Date; + +/** + * 小区周边配套统计信息 + * + * @author shenjian@fangjia.com + */ +@Entity +@Table(name = "fangjia_stat.community_poi_gj") +public class CommunityPoi implements MongoEntity { + + private static final long serialVersionUID = -7663644539542318409L; + + private String id; + private String city; + private String region; + private String name; + private Integer subway; + private Integer traffic; + private Integer bank; + private Integer primary; + private Integer junior; + private Integer hospital; + /** + * 不利设施,丧葬 + */ + private Integer funeral; + + @Id + @Column(name = "_id") + @Override + public String getId() { + return id; + } + + @Override + public void setId(String id) { + this.id = id; + } + + @Column(name = "city") + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + @Column(name = "region") + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + @Column(name = "name") + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + @Column(name = "subway") + public Integer getSubway() { + return subway; + } + + public void setSubway(Integer subway) { + this.subway = subway; + } + + @Column(name = "traffic") + public Integer getTraffic() { + return traffic; + } + + public void setTraffic(Integer traffic) { + this.traffic = traffic; + } + + @Column(name = "bank") + public Integer getBank() { + return bank; + } + + public void setBank(Integer bank) { + this.bank = bank; + } + + @Column(name = "primary") + public Integer getPrimary() { + return primary; + } + + public void setPrimary(Integer primary) { + this.primary = primary; + } + + @Column(name = "junior") + public Integer getJunior() { + return junior; + } + + public void setJunior(Integer junior) { + this.junior = junior; + } + + @Column(name = "hospital") + public Integer getHospital() { + return hospital; + } + + public void setHospital(Integer hospital) { + this.hospital = hospital; + } + + @Column(name = "funeral") + public Integer getFuneral() { + return funeral; + } + + public void setFuneral(Integer funeral) { + this.funeral = funeral; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index e71226a4a..69108673a 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -67,6 +67,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i must.add(addTerms("district_name", alias)); must.add(addMatch("cat", "sell")); must.add(addRange("avg_price", minPrice, maxPrice)); + must.add(addRangeGte("p_date", 20210501)); + bool.put("must", must); query.put("bool", bool); reqBody.put("query", query); @@ -161,6 +163,16 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i return term; } + private JSONObject addRangeGte(String name, Number min){ + JSONObject term = new JSONObject(); + JSONObject range = new JSONObject(); + JSONObject rangeValue = new JSONObject(); + rangeValue.put("gte", min); + range.put(name, rangeValue); + term.put("range", range); + return term; + } + private List wrapSeawater(List jsonList){ List seawaterList= new ArrayList(); @@ -203,6 +215,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i seawater.setFloor(source.getInt("floor")); }catch(Exception e){ seawater.setFloor(0); + } + try{ + seawater.setFloorInfo(source.getString("floor_info")); + }catch(Exception e){ + } if(source.containsKey("fitment")){ seawater.setFitment(source.getString("fitment")); @@ -231,6 +248,14 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(source.containsKey("root")){ seawater.setRoot(source.getString("root")); } + + if(source.containsKey("estate_type")){ + seawater.setEstateType(source.getString("estate_type")); + } + + if(source.containsKey("build_type")){ + seawater.setBuildType(source.getString("build_type")); + } seawaterList.add(seawater); } return seawaterList; @@ -379,11 +404,11 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i } // 不展示删除 - JSONObject param = new JSONObject(); - param.put("status", 0); - JSONObject temp = new JSONObject(); - temp.put("term", param); - jsonList.add(temp); +// JSONObject param = new JSONObject(); +// param.put("status", 0); +// JSONObject temp = new JSONObject(); +// temp.put("term", param); +// jsonList.add(temp); query.put("must", jsonList); q.put("bool", query); diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java index c761ba76a..1af563193 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/model/Seawater.java @@ -48,6 +48,7 @@ public class Seawater implements SeawaterEntity { private Double area; private Integer height; private Integer floor; + private String floorInfo; private String fitment; private Integer room; private Integer hall; @@ -58,6 +59,9 @@ public class Seawater implements SeawaterEntity { private String url; private String root; private Integer avgPrice; + private String purpose; + private String estateType; + private String buildType; @Id @Column(name = "_id") @@ -195,6 +199,15 @@ public class Seawater implements SeawaterEntity { this.floor = floor; } + @Column(name = "floorInfo") + public String getFloorInfo() { + return floorInfo; + } + + public void setFloorInfo(String floorInfo) { + this.floorInfo = floorInfo; + } + @Column(name = "fitment") public String getFitment() { return fitment; @@ -285,4 +298,31 @@ public class Seawater implements SeawaterEntity { public void setAvgPrice(Integer avgPrice) { this.avgPrice = avgPrice; } + + @Column(name = "purpose") + public String getPurpose() { + return purpose; + } + + public void setPurpose(String purpose) { + this.purpose = purpose; + } + + @Column(name = "estate_type") + public String getEstateType() { + return estateType; + } + + public void setEstateType(String estateType) { + this.estateType = estateType; + } + + @Column(name = "build_type") + public String getBuildType() { + return buildType; + } + + public void setBuildType(String buildType) { + this.buildType = buildType; + } } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java index c5991df90..8283df958 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java @@ -102,6 +102,8 @@ public class FjtEvaluation implements SqlEntity{ private String finishDateStr;//竣工时间 private String useType;//房屋用途 + + private Date downPdfDate;//下载报告时间 @GeneratedValue(strategy = GenerationType.AUTO) @Id @@ -750,6 +752,16 @@ public class FjtEvaluation implements SqlEntity{ this.useType = useType; } + + @Column(name = "down_pdf_date") + public Date getDownPdfDate() { + return downPdfDate; + } + + public void setDownPdfDate(Date downPdfDate) { + this.downPdfDate = downPdfDate; + } + @Override public String toString() { return "FjtEvaluation{" + diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/OpenConfig.java b/projects/fangjia-open/src/main/java/com/fangjia/open/OpenConfig.java index e1064e8cd..c13a28b36 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/OpenConfig.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/OpenConfig.java @@ -38,6 +38,7 @@ public class OpenConfig extends Configuration { private final String ipdatfile; private final String baiduAk; + private final String defaultCity; private final boolean isOpenAllAuthority; @@ -54,6 +55,7 @@ public class OpenConfig extends Configuration { baiduAk = configuration.get("fangjia.baidu.ak"); isOpenAllAuthority = configuration.getBoolean("fangjia.isOpenAllAuthority", false); enabledAddressMatch = configuration.getBoolean("fangjia.addressMatch.enabled",false); + defaultCity = configuration.get("fangjia.default.city"); } public static OpenConfig getConfig() { @@ -134,4 +136,7 @@ public class OpenConfig extends Configuration { return enabledAddressMatch; } + public String getDefaultCity() { + return defaultCity; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 907212d5f..af890d880 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -89,6 +89,7 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService; import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; +import org.sothis.web.mvc.annotation.Param; @Authority(role = Role.USER) public class AddressController { @@ -106,6 +107,7 @@ public class AddressController { private HouseLoudongService houseLoudongService; private FjtEvaluationService fjtEvaluationService; private AddressMatchProxyService addressMatchProxyService; + private final static String AK = OpenConfig.getConfig().getBaiduAk(); private final static Logger LOGGER = LoggerFactory.getLogger(AddressController.class); @@ -117,6 +119,8 @@ public class AddressController { private static final String GRADE_NATION = "全国评分"; private static final String GRADE_CONSUME = "消费评分"; + public final static OpenConfig CONFIG = OpenConfig.getConfig(); + /** * @deprecated see parse2Action * @param model @@ -1705,6 +1709,228 @@ public class AddressController { return mav; } + /** + * 地址标准化接口V2 + * + * @param model + * @return + */ + public Object standard2Action(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 五级地址标准化接口 + * + * @param model + * @return + */ + public Object fiveDivisionStructureAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + // 地址验证无效也输出标准化结果 + params.put("flag", "true"); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 五级行政区划V2地址标准化接口 + * + * @param model + * @return + */ + public Object fiveDivisionStructure2Action(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + // 地址验证无效也输出标准化结果 + params.put("flag", "true"); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 详细地址标准化接口 + * + * @param model + * @return + */ + public Object divisionStructureAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + // 地址验证无效也输出标准化结果 + params.put("flag", "true"); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 地址标准化专用测试接口 + * @param model + * @return + */ + public Object structureAndCompleteAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + // 地址验证无效也输出标准化结果 + params.put("flag", "true"); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + public Object similarityAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java index 13e59faa4..b515c4593 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AuctionController.java @@ -1,6 +1,7 @@ package com.fangjia.open.controllers; import com.fangjia.core.util.DateUtil; +import com.fangjia.core.util.HttpRequest; import com.fangjia.dal.QueryParam; import com.fangjia.dal.SortParam; import com.fangjia.dal.common.DistrictKey; @@ -14,6 +15,7 @@ import com.fangjia.open.model.BasicModel; import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.service.fsh.DistrictNewPriceService; import com.fangjia.service.open.PaiMaiService; +import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.bson.types.ObjectId; @@ -23,13 +25,14 @@ import java.util.regex.Pattern; /** * 法拍数据提供接口 */ -@Authority(role = Role.USER) +//@Authority(role = Role.USER) public class AuctionController { private PaiMaiService paiMaiService; private DistrictNewPriceService districtNewPriceService; - private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", "matchEstateType"}; + private static final String[] SEARCH_FIELDS = new String[]{"marketAvgPrice", "startDate", "curPrice", "imgUrls", "id", "matchCity", "matchRegion", "matchName", "address", "marketTotalPrice", "startPrice", "curPrice", "predictPrice", "depositPrice", "area", "status", "url", + "matchEstateType","type","count", "tradeAvgPrice", "evaluationPrice"}; public Object housesAction(AuctionModel model){ RestfulModelAndView mav = new RestfulModelAndView(); @@ -187,6 +190,79 @@ public class AuctionController { return mav; } + + public Object searchPaimaisAction(AuctionModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + List params = new ArrayList(); + if(StringUtils.isNotBlank(model.getCity())){ + params.add(new QueryParam("matchCity", model.getCity())); + } + if(StringUtils.isNotBlank(model.getDistrict())){ + params.add(new QueryParam("matchRegion", model.getDistrict())); + } + + if(StringUtils.isNotBlank(model.getName())){ + params.add(new QueryParam("matchName", model.getName())); + } + + Date startDate = null; + if(StringUtils.isNotBlank(model.getStartDate())){ + startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); + if(startDate != null){ + params.add(new QueryParam("startDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate)); + } + } + + List sortParams = new ArrayList(); + sortParams.add(new SortParam("startDate", false)); + List paimais = paiMaiService.findAll(params, sortParams, model.getLimit(), SEARCH_FIELDS); + if(paimais!=null && paimais.size()>0){ + for(PaiMai paiMai : paimais){ + Map map = standard(paiMai.getCity(), paiMai.getAddress()); + if(map!=null){ + if (map.containsKey("buildingNumber")) { + paiMai.setHouseNum((String) map.get("buildingNumber")); + } + if (map.containsKey("unitNumber")) { + paiMai.setUnitNum((String) map.get("unitNumber")); + } + if (map.containsKey("roomNumber")) { + paiMai.setRoomNum((String) map.get("roomNumber")); + } + } + } + } + mav.setModel(BasicModel.buildOKModel().setResult(paimais)); + return mav; + } + + + private Map standard(String city, String address){ + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + if(json.containsKey("result")){ + return (Map)json.get("result"); + } + } + return null; + } + public void setPaiMaiService(PaiMaiService paiMaiService) { this.paiMaiService = paiMaiService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java index 8abece9e2..ab916308a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DealPriceController.java @@ -15,8 +15,10 @@ import java.util.Map.Entry; import java.util.regex.Matcher; import java.util.regex.Pattern; +import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.mongo.model.Metro; import com.fangjia.open.model.BasicModel; +import com.fangjia.service.bo.DistrictStatBo; import org.apache.commons.lang.StringUtils; import com.fangjia.core.util.DateUtil; @@ -211,6 +213,42 @@ public class DealPriceController { mv.setModel(resMap); return mv; } + + public Object searchListAction(final DealPriceModel model) { + RestfulModelAndView mv = new RestfulModelAndView(); + mv.setViewType("rest"); + + if (StringUtils.isBlank(model.getId())) { + mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空")); + return mv; + } + District district = districtService.findDistrictById(model.getId()); + DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + + if(district == null){ + mv.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mv; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + Date startDate = null; + Date endDate = null; + if(StringUtils.isNotBlank(model.getStartDate())){ + startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); + } + if(StringUtils.isNotBlank(model.getEndDate())){ + endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd"); + } + + List dealPrices = dealPriceService.getNearlySameDealPrice(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit()); + mv.setModel(BasicModel.buildOKModel().setResult(dealPrices)); + return mv; + } private List findDealPricesByName(String city,String region ,String name){ Calendar calendar = Calendar.getInstance(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java index 859a04ef0..395cb8a8d 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/DistrictController.java @@ -7,11 +7,11 @@ import javax.servlet.http.HttpServletRequest; import com.fangjia.core.RegionData; import com.fangjia.core.util.DateUtil; -import com.fangjia.dal.mongo.model.RegionStat; -import com.fangjia.dal.mongo.model.RegionStatRent; +import com.fangjia.dal.mongo.model.*; import com.fangjia.open.model.AvgPriceQueryModel; import com.fangjia.open.model.BasicModel; import com.fangjia.service.datastore.RegionStatService; +import com.fangjia.service.district.DistrictFacilitysService; import com.fangjia.service.district.DistrictStatRentService; import com.fangjia.service.district.RegionStatRentService; import com.fangjia.service.mongo.fangjia.AverageAnalyService; @@ -38,8 +38,6 @@ import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.FangjiaServerAreaContextHolder; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; -import com.fangjia.dal.mongo.model.DistrictHouse; -import com.fangjia.dal.mongo.model.DistrictStat; import com.fangjia.dal.mongo.model.bkoff.ExtractDistrict; import com.fangjia.dal.seaweed.model.District; import com.fangjia.dal.sql.model.ApiUser; @@ -93,6 +91,7 @@ public class DistrictController { private RegionBlockService regionBlockService; private RegionStatService regionStatService; private RegionStatRentService regionStatRentService; + private DistrictFacilitysService districtFacilitysService; public static final String[] COLUMNS = { "id", "name", "estateType", "buildType", "licence", "buildArea", "allArea", @@ -444,19 +443,19 @@ public class DistrictController { * * @return */ - public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, - @Param(name = "name") String name, HttpServletRequest request) { - RestfulModelAndView mav = new RestfulModelAndView(); - mav.setViewType("rest"); - if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { - mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); - return mav; - } else { - DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name)); - if (null == districtBo) { - mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件")); + public Object showAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, + @Param(name = "name") String name, HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); return mav; - } + } else { + DistrictBo districtBo = districtService.findByFullMode(new DistrictKey(city, region, name)); + if (null == districtBo) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在,请检查查询条件")); + return mav; + } districtBo.setStat(this.districtStatService.findDistrictStat(districtBo.getDistrictKey())); // 去除小区描述中的html标签 districtBo.setDescription(DataUtil.removeSpecialString(districtBo.getDescription())); @@ -1146,6 +1145,25 @@ public class DistrictController { ret.put("RESULT", 200); return mav; } + + public Object facilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "region") String region, + @Param(name = "name") String name, HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(name) || StringUtils.isBlank(city) || StringUtils.isBlank(region)) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "小区所在城市,小区名,所在区域必必须输入")); + return mav; + } else { + District districtBo = districtService.findDistrict(new DistrictKey(city, region, name), "district"); + if (null == districtBo) { + mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在")); + return mav; + } + DistrictFacilitys facilitys = districtFacilitysService.findByDistrict(districtBo); + mav.setModel(BasicModel.buildOKModel().setResult(facilitys)); + } + return mav; + } @@ -1230,4 +1248,7 @@ public class DistrictController { this.regionStatRentService = regionStatRentService; } + public void setDistrictFacilitysService(DistrictFacilitysService districtFacilitysService) { + this.districtFacilitysService = districtFacilitysService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java index 67e7b084c..7ed1a83f4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/EvaluationController.java @@ -10,9 +10,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import com.fangjia.dal.mongo.model.BsEvaluation; +import com.fangjia.dal.mongo.model.bkoff.DealPrice; import com.fangjia.service.bo.AddrEvaResult; +import com.fangjia.service.bo.DistrictStatBo; import com.fangjia.service.bo.EvaluationBo; import com.fangjia.service.bo.HouseScorer; +import com.fangjia.service.datastore.*; +import com.fangjia.service.district.DistrictStatService; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; @@ -54,11 +59,6 @@ import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.service.api.AutoCompleteService; import com.fangjia.service.api.KnowbaseService; import com.fangjia.service.common.SellSimilar; -import com.fangjia.service.datastore.BusinessEstateService; -import com.fangjia.service.datastore.DistrictService; -import com.fangjia.service.datastore.EvaluationReportService; -import com.fangjia.service.datastore.EvaluationService; -import com.fangjia.service.datastore.HouseService; import com.fangjia.service.sql.ApiUserService; import com.fangjia.service.sql.UserService; import com.fangjia.service.web.AddressService; @@ -94,6 +94,8 @@ public class EvaluationController { private ApiUserService apiUserService; private UserService userService; private AddressService addressService; + private DistrictStatService districtStatService; + private BsEvaluationService bsEvaluationService; private final static String AK = OpenConfig.getConfig().getBaiduAk(); public Object lngLatEvaluateAction(EvaluationModel model) { @@ -1005,6 +1007,44 @@ public class EvaluationController { return mav; } + public Object bsListAction(EvaluationModel model) { + RestfulModelAndView mv = new RestfulModelAndView(); + mv.setViewType("rest"); + + if (StringUtils.isBlank(model.getId())) { + mv.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "楼盘编号不能为空")); + return mv; + } + District district = districtService.findDistrictById(model.getId()); + DistrictKey districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + + if(district == null){ + mv.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mv; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + Date startDate = null; + Date endDate = null; + if(StringUtils.isNotBlank(model.getStartDate())){ + startDate = DateUtil.parseDate(model.getStartDate(),"yyyyMMdd"); + } + if(StringUtils.isNotBlank(model.getEndDate())){ + endDate = DateUtil.parseDate(model.getEndDate(),"yyyyMMdd"); + } + + List dealPrices = bsEvaluationService.getNearlySameEvaluation(model.getId(), minPrice, maxPrice, startDate, endDate, model.getLimit()); + mv.setModel(BasicModel.buildOKModel().setResult(dealPrices)); + return mv; + } + + + public void setEvaluationService(EvaluationService evaluationService) { this.evaluationService = evaluationService; } @@ -1017,6 +1057,18 @@ public class EvaluationController { this.knowbaseService = knowbaseService; } + + + public void setDistrictStatService(DistrictStatService districtStatService) { + this.districtStatService = districtStatService; + } + + + + public void setBsEvaluationService(BsEvaluationService bsEvaluationService) { + this.bsEvaluationService = bsEvaluationService; + } + public void setAutoCompleteService(AutoCompleteService autoCompleteService) { this.autoCompleteService = autoCompleteService; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java index ebb6c9025..c75ec0da7 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/HouseController.java @@ -11,6 +11,8 @@ import com.fangjia.dal.mongo.model.DistrictFacilitys; import com.fangjia.dal.mongo.model.FangjiaOfficialAreaMap; import com.fangjia.dal.mongo.model.Metro; import com.fangjia.dal.mongo.model.StatsIndex; +import com.fangjia.dal.seawater.dao.SeawaterDao; +import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seaweed.model.District; import com.fangjia.dal.sql.model.ApiUser; import com.fangjia.datastore.driver.FacetResult; @@ -29,6 +31,7 @@ import com.fangjia.open.ws.query.CollateralSampleQuery; import com.fangjia.open.ws.query.ShopCollateralSampleQuery; import com.fangjia.service.address.AddressParser; import com.fangjia.service.bo.Collateral; +import com.fangjia.service.bo.DistrictStatBo; import com.fangjia.service.bo.Sample; import com.fangjia.service.bo.ShopSample; import com.fangjia.service.datastore.CzbankSampleService; @@ -39,11 +42,13 @@ import com.fangjia.service.datastore.bean.CollateralSampleQueryBean; import com.fangjia.service.datastore.bean.ShopCollateralSampleQueryBean; import com.fangjia.service.district.DistrictFacilitysService; import com.fangjia.service.district.DistrictStatService; +import com.fangjia.service.district.SqmetreRentService; import com.fangjia.service.menu.Item; import com.fangjia.service.menu.MenuCategoryEnum; import com.fangjia.service.menu.Value; import com.fangjia.service.mongo.fangjia.DealPriceService; import com.fangjia.service.mongo.fangjia.MetroService; +import com.fangjia.service.mongo.fangjia.SeawaterService; import com.fangjia.service.mongo.fangjia.StatsIndexService; import com.fangjia.service.open.FangjiaOfficialAreaService; import com.fangjia.service.open.HousePojo; @@ -86,6 +91,9 @@ public class HouseController { private CzbankSampleService czbankSampleService; private DistrictService districtService; private FangjiaOfficialAreaService fangjiaOfficialAreaService; + + private SeawaterService seawaterService; + private SeawaterDao seawaterDao; private List PROPERTY_TYPE_CODE_LIST = Arrays.asList("01", "02"); @@ -399,6 +407,107 @@ public class HouseController { } } + public Object listAction(final HouseSearchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) { + mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项")); + return mav; + } + District district = null; + DistrictKey districtKey = null; + if(StringUtils.isNotBlank(model.getName())){ + districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName()); + district = districtService.findDistrict(districtKey, "district"); + }else{ + district = districtService.findDistrictById(model.getId()); + if(district!=null){ + districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + } + } + + if(district == null){ + mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mav; + } + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + List names = new ArrayList(); + names.add(district.getName()); + if(district.getAlias() != null){ + for(String alia : district.getAlias()){ + if(StringUtils.isNotBlank(alia) && !names.contains(alia)){ + names.add(alia); + } + } + } + List houses = seawaterDao.findSeawatersFromEs("sell", district.getCity(), district.getRegion(), district.getName(), district.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater"); + //List houses1 = seawaterDao.findHouses(district.getCity(), district.getRegion(), names, minPrice, maxPrice, model.getPageSize()); + mav.setModel(BasicModel.buildOKModel().setResult(houses)); + return mav; + } + + + + public Object arroundlistAction(final HouseSearchModel model) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(model.getId()) && (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getRegion()) || StringUtils.isBlank(model.getName()))) { + mav.setModel(BasicModel.buildBadRequestModel("楼盘编号或城市,区域,名称为必填项")); + return mav; + } + District district = null; + DistrictKey districtKey = null; + if(StringUtils.isNotBlank(model.getName())){ + districtKey = new DistrictKey(model.getCity(), model.getRegion(), model.getName()); + district = districtService.findDistrict(districtKey, "district"); + }else{ + district = districtService.findDistrictById(model.getId()); + if(district!=null){ + districtKey = new DistrictKey(district.getCity(), district.getRegion(), district.getName()); + } + } + + if(district == null){ + mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mav; + } + + //周边小区,默认10 + List districts = districtService.findNearestDistrict(district, 1000, model.getDistCount()); + List houses = new ArrayList(); + for(District dist : districts){ + districtKey = new DistrictKey(dist.getCity(), dist.getRegion(), dist.getName()); + DistrictStatBo bo = districtStatService.findCurrentDistrictStat(districtKey); + Integer minPrice = null; + Integer maxPrice = null; + if(bo.getDistrictStat()!=null){ + minPrice = (int)(bo.getDistrictStat().getAvgPrice() * 0.8); + maxPrice = (int)(bo.getDistrictStat().getAvgPrice() * 1.2); + } + List names = new ArrayList(); + names.add(district.getName()); + if(district.getAlias() != null){ + for(String alia : district.getAlias()){ + if(StringUtils.isNotBlank(alia) && !names.contains(alia)){ + names.add(alia); + } + } + } + List arroundhouses = seawaterDao.findSeawatersFromEs("sell", dist.getCity(), dist.getRegion(), dist.getName(), dist.getAlias(), null, model.getStartTime(), model.getEndTime(),null,null,minPrice, maxPrice,null,null,"p_date", false,0, model.getPageSize(),"latest_seawater"); + if(arroundhouses!=null && arroundhouses.size()>0){ + houses.addAll(arroundhouses); + } + } + mav.setModel(BasicModel.buildOKModel().setResult(houses)); + return mav; + } + String getPicUrl(String url, List urls) { if (!urls.contains(url)) { urls.add(url); @@ -832,6 +941,13 @@ public class HouseController { return ret; } + public void setSeawaterService(SeawaterService seawaterService) { + this.seawaterService = seawaterService; + } + + public void setSeawaterDao(SeawaterDao seawaterDao) { + this.seawaterDao = seawaterDao; + } private boolean checkExists(FacetQueryResult houseBoList, HousePojo house) { for (HousePojo bo : houseBoList) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 1ba157ee5..1874f3500 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -31,12 +31,15 @@ import javax.validation.Valid; import com.fangjia.core.util.*; import com.fangjia.dal.*; import com.fangjia.dal.mongo.model.*; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; import com.fangjia.dal.seawater.dao.SeawaterDao; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.open.dto.*; import com.fangjia.service.district.*; import com.fangjia.service.mongo.fangjia.*; import com.fangjia.service.open.*; +import com.fangjia.service.open.impl.CommunityPoiServiceImpl; import com.fangjia.service.util.DistrictScoreUtil; import net.sf.json.JSONArray; import net.sf.json.JSONObject; @@ -175,6 +178,9 @@ public class PropertyController { private SeawaterDao seawaterDao; private SeawaterService seawaterService; private SqmetreRentService sqmetreRentService; + private CommunityPoiService communityPoiService; + private CommunityBasePriceService communityBasePriceService; + private EvaluateFactorService evaluateFactorService; private FileDao fileDao = FileDao.getInstance(); private static final int DEFAULT_PAGE_SIZE = 20; @@ -390,10 +396,31 @@ public class PropertyController { put("5000", "five");// 五线 } }; + /** + * 第三方均价映射 + */ + private static final Map OTHER_PRICE_MAP = new HashMap() { + { + put("zhuge_price", "诸葛找房"); + put("anjuke_price", "安居客"); + put("fang_price", "房天下"); + put("lianjia_price", "链家"); + put("cityhouse_price", "禧泰数据"); + put("maitian_price", "麦田"); + put("ke_price", "贝壳"); + put("taofang_price", "淘房365"); + put("zhongyuan_price", "中原"); + put("tbs_price", "兔博士"); + put("qfang_price", "Q房"); + put("fangtan_price", "房探"); + put("fangduoduo_price", "房多多"); + put("wiwj_price", "我爱我家"); + } + }; /** * 利用百度API把GPS转换成Google经纬度 - * + * * @param lng * @param lat * @return @@ -415,7 +442,7 @@ public class PropertyController { /** * 利用算法模板把GPS转换成Google经纬度 - * + * * @param lng * @param lat * @return @@ -607,7 +634,7 @@ public class PropertyController { /** * 判断小区是否有经纬度 - * + * * @param district * @return */ @@ -618,7 +645,7 @@ public class PropertyController { /** * 调用匹配接口匹配小区 - * + * * @param city * @param address * @param region @@ -666,17 +693,6 @@ public class PropertyController { } String userId = FangjiaApiUtil.parseUserId(model.getToken()); - /**超级用户和上海交大账户不控制权限*/ - if (!"1007497".equals(userId) && !"1007632".equals(userId) && !"1007548".equals(userId) - && !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId) - && !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId) - && !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId) - && !"1007747".equals(userId) && !"1007757".equals(userId) - && model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制 - mav.setModel(BasicModel.buildUnAuthorizedModel()); - return mav; - } - District district = null; if (StringUtils.isNotBlank(model.getId())){ if(ObjectId.isValid(model.getId())){ @@ -740,18 +756,18 @@ public class PropertyController { } else { districtInfoDto.setBuildingType(""); } - districtInfoDto.setAlias(district.getAlias()); + districtInfoDto.setAlias(district.getMappingAlias()); districtInfoDto.setAddressAlias(district.getAddrAlias()); districtInfoDto.setCommunity(district.getStreetCommunity()); districtInfoDto.setCommercialArea(district.getBusinessQuarter()); if (district.getHouseholdCount() != null) { - districtInfoDto.setTotalUnitCount(district.getHouseholdCount().toString()); + districtInfoDto.setTotalUnitCount(SeaweedUtils.formatHouseholdCount(district.getHouseholdCount().toString())); } else { districtInfoDto.setTotalUnitCount(""); } districtInfoDto.setManagementFee(district.getEstateCharge()); if (district.getCarSiteCount() != null) { - districtInfoDto.setParking(district.getCarSiteCount().toString()); + districtInfoDto.setParking(SeaweedUtils.formatCarSiteCount(district.getCarSiteCount().toString())); } else { districtInfoDto.setParking(""); } @@ -772,10 +788,10 @@ public class PropertyController { districtInfoDto.setFinishBuildTime(district.getFinishBuildingDate()); districtInfoDto.setMoveinTime(district.getLiveDate()); - districtInfoDto.setFloorSpace(district.getAllArea()); + districtInfoDto.setFloorSpace(SeaweedUtils.formatAllArea(district.getAllArea())); districtInfoDto.setConstructionSpace(district.getBuildArea()); - districtInfoDto.setPlotRatio(district.getCubagePercent()); - districtInfoDto.setGreenPercentage(district.getVirescencePercent()); + districtInfoDto.setPlotRatio(SeaweedUtils.formatCubagePercent(district.getCubagePercent())); + districtInfoDto.setGreenPercentage(SeaweedUtils.formatVirescencePercent(district.getVirescencePercent())); districtInfoDto.setPreSalePermit(district.getLicence()); districtInfoDto.setDeveloper(district.getDeveloper()); districtInfoDto.setDiscount(district.getDiscount()); @@ -792,6 +808,7 @@ public class PropertyController { districtInfoDto.setDescription(district.getDescription()); districtInfoDto.setSalesOfficeAdd(district.getSellAddress()); districtInfoDto.setSalesPhoneNumber(district.getMobile()); + districtInfoDto.setTownship(district.getTownship()); if (district.getLoudongCount() != null && district.getLoudongCount() > 0) { districtInfoDto.setBuildingCount(district.getLoudongCount()); } @@ -960,6 +977,12 @@ public class PropertyController { return mav; } + if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){ + Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber()); + if(loudong != null && loudong.getFloors() >= model.getFloor()){ + model.setMaxFloor(loudong.getFloors()); + } + } model.setRegion(district.getRegion()); model.setDistrictName(district.getName()); if (StringUtils.isBlank(model.getPropertyType())) { @@ -996,6 +1019,32 @@ public class PropertyController { } } + // 调整面积不同面积价格 + try { + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } + } + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95); + Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05); + // 偏离度超过5%不进行调整 + if (minPrice <= avgPrice && avgPrice <= maxPrice) { + evaluation.setAvgPrice(avgPrice); + evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); + } + } + } catch (Exception ex) { + + } + if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0 && (cityPriceCredit == null || evaluation.getCredit() >= cityPriceCredit)) { Map districtPriceMap = new HashMap(); @@ -1116,7 +1165,7 @@ public class PropertyController { model.setCalDate(calDate); } /* 先到CPDB库中查找小区 */ - District district = matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName()); + District district = districtService.findDistrict(new DistrictKey(model.getCity(), model.getDistrict(), model.getName()), "district");//matchDistrict(model.getCity(), model.getAddress(), model.getDistrict(), model.getName()); if(district == null){ //防止地址模糊,名称详细的情况 district = matchDistrict(model.getCity(), model.getName(), model.getDistrict(), model.getAddress()); @@ -1166,19 +1215,52 @@ public class PropertyController { evaluationService.remove(evaluation.getId()); mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); return mav; + } + // 调整面积不同面积价格 + try { + DistrictStatBo districtStatBo = districtStatService.findDistrictStat(new DistrictKey(district)); + int orgAvgPrice = 0; + if (districtStatBo != null) { + if (StringUtils.isNotBlank(district.getEstateType2()) && district.getEstateType2().contains("别墅") && districtStatBo.getVillaDistrictStat() != null) { + orgAvgPrice = districtStatBo.getVillaDistrictStat().getAvgPrice(); + } else if (districtStatBo.getDistrictStat() != null) { + orgAvgPrice = districtStatBo.getDistrictStat().getAvgPrice(); + } + } + if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) { + int factorPrice = evaluation.getAvgPrice(); + int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice); + Integer minPrice = (int)(evaluation.getAvgPrice() * 0.95); + Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.05); + // 偏离度超过5%不进行调整 + if (minPrice <= avgPrice && avgPrice <= maxPrice) { + evaluation.setAvgPrice(avgPrice); + evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea())); + } + } + } catch (Exception ex) { + } Map districtPriceMap = new LinkedHashMap(); districtPriceMap.put("city", district.getCity()); districtPriceMap.put("district", district.getRegion()); districtPriceMap.put("name", district.getName()); long totalPrice = evaluation.getBigTotalPrice(); - if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){ - totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity()))); - evaluation.setBigTotalPrice(totalPrice); - evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea())); - } + // 取消系数下调 +// if(CITY_CORRECTION_FACTOR.containsKey(district.getCity())){ +// totalPrice = (long)(totalPrice * (1 - CITY_CORRECTION_FACTOR.get(district.getCity()))); +// evaluation.setBigTotalPrice(totalPrice); +// evaluation.setAvgPrice((int)(totalPrice/evaluation.getArea())); +// } districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice()); districtPriceMap.put("unitPrice", evaluation.getAvgPrice()); + // 新增置信度 + CommunityBasePrice communityBasePrice = communityBasePriceService.getById(district.getId()); + if (communityBasePrice != null) { + districtPriceMap.put("credit", communityBasePrice.getCredit()); + } else { + districtPriceMap.put("credit", evaluation.getCredit()); + } List names = new ArrayList(); names.add(district.getName()); if(district.getAlias() != null){ @@ -1190,6 +1272,31 @@ public class PropertyController { } Integer minPrice = (int)(evaluation.getAvgPrice() * 0.8); Integer maxPrice = (int)(evaluation.getAvgPrice() * 1.2); + Integer min10Price = (int)(evaluation.getAvgPrice() * 0.9); + Integer max10Price = (int)(evaluation.getAvgPrice() * 1.1); + // 新增第三方均价 + try { + double dCredit = 0.61; + Map rMap = getLastEvaluationPrice(district, evaluation, dCredit); + if (null != rMap) { + if (rMap.containsKey("other_price")) { + Map otherPriceMap = (Map) rMap.get("other_price"); + Map priceResultMap = new HashMap(); + for (String key : otherPriceMap.keySet()) { + Integer price = otherPriceMap.get(key); + // 剔除偏差大于10的 + if (min10Price <= price && max10Price >= price) { + if (OTHER_PRICE_MAP.containsKey(key)) { + priceResultMap.put(OTHER_PRICE_MAP.get(key), price); + } + } + } + districtPriceMap.put("otherPrice", priceResultMap); + } + } + } catch (Exception ex) { + + } List houses = seawaterDao.findHouses(district.getCity(), district.getRegion(), names,minPrice, maxPrice, 20); if(houses.size() > 3){ districtPriceMap.put("calcType", 1); @@ -1218,7 +1325,7 @@ public class PropertyController { }else if(district.getLng() != null && district.getLng() > 0){ double lng = DataUtil.roundDouble(district.getLng() / 10000000000.0,6); double lat = DataUtil.roundDouble(district.getLat() / 10000000000.0, 6); - List districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 4, null); + List districts = districtService.findNearestDistrict(district.getEstateType2(), lng, lat, 1000 * 3 , 6, null); if(districts.size() < 2){ evaluationService.remove(evaluation.getId()); mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的估价").markNoData()); @@ -1256,9 +1363,15 @@ public class PropertyController { districtDto.append("buildYear", String.valueOf(buildYear)); } districtAnalyData = districtAnalyDataService.findByDistrictKey(new DistrictKey(d)); - if(districtAnalyData != null && districtAnalyData.getHalfYearMarkup() != null){ - districtDto.append("halfYearMarkup",DataUtil.roundDouble( districtAnalyData.getHalfYearMarkup() * 100 ,2)); + if(districtAnalyData != null){ + if (districtAnalyData.getHalfYearMarkup() != null) { + districtDto.append("halfYearMarkup", DataUtil.roundDouble(districtAnalyData.getHalfYearMarkup() * 100, 2)); + } + if (districtAnalyData.getAvgPrice() != null && minPrice <= districtAnalyData.getAvgPrice() && maxPrice >= districtAnalyData.getAvgPrice()) { + districtDto.append("avgPrice", districtAnalyData.getAvgPrice()); + } } + nearCommunityDtos.add(districtDto); } Integer buildYear = districtService.getBuildYear(district); @@ -1266,6 +1379,18 @@ public class PropertyController { modelFeaturesFto.put("buildYear", String.valueOf(buildYear)); } modelFeaturesFto.append("nearCommunity", nearCommunityDtos); + + // 模型特征新增周边数量 + CommunityPoi communityPoi = communityPoiService.getById(district.getId()); + if (communityPoi != null) { + modelFeaturesFto.put("subway", communityPoi.getSubway()); + modelFeaturesFto.put("traffic", communityPoi.getTraffic()); + modelFeaturesFto.put("bank", communityPoi.getBank()); + modelFeaturesFto.put("primary", communityPoi.getPrimary()); + modelFeaturesFto.put("hospital", communityPoi.getHospital()); + modelFeaturesFto.put("junior", communityPoi.getJunior()); + modelFeaturesFto.put("funeral", communityPoi.getFuneral()); + } districtPriceMap.put("modelFeatures", modelFeaturesFto); } mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap)); @@ -1473,7 +1598,7 @@ public class PropertyController { return mav; } } - + /** * 四川锦程 定制接口 多建筑年代 * @param model @@ -1585,7 +1710,7 @@ public class PropertyController { districtPriceMap.put("buildYear", matcher3.group()); } } - + // 根据城市等 级设置D级可信度 ApiUserAuthority apiUserAuthority = apiUserAuthorityService @@ -2098,7 +2223,7 @@ public class PropertyController { /** * 小区租金查询接口 - * + * * @param model * @return */ @@ -2363,7 +2488,7 @@ public class PropertyController { mav.setModel(BasicModel.buildBadRequestModel("size(面积)不能为空")); return mav; } - District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(), + District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName().replace(" ",""), "district"); if (null == district) { mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); @@ -2577,7 +2702,7 @@ public class PropertyController { /** * 批量获取小区的均价,均价走势数据 - * + * * @param request * @return */ @@ -2719,7 +2844,7 @@ public class PropertyController { /** * 批量获取小区的配套数据,最多12个小区 - * + * * @param request * @return */ @@ -2864,7 +2989,7 @@ public class PropertyController { /** * 获取JSON参数中的num参数值 - * + * * @param jo * @return */ @@ -2889,7 +3014,7 @@ public class PropertyController { /** * 获取JSON参数中的radius值 - * + * * @param jo * @return */ @@ -3697,7 +3822,7 @@ public class PropertyController { /** * 解析出关键字中的小区名 - * + * * @param keyword * @return */ @@ -3814,7 +3939,7 @@ public class PropertyController { /** * 小区统计分析接口 - * + * * @param model * @return */ @@ -3899,7 +4024,7 @@ public class PropertyController { /** * 小区交通接口 - * + * * @param model * @return */ @@ -3957,7 +4082,7 @@ public class PropertyController { /** * 小区周边接口 - * + * * @param model * @return */ @@ -4016,7 +4141,7 @@ public class PropertyController { /** * 相同小区相似户型房源挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源 - * + * * @author yinjihuan * @date 2016年9月1日 * @param city @@ -4056,7 +4181,7 @@ public class PropertyController { /** * 周边小区相似户型挂牌量接口
* 获取该小区的满足指定相似条件的挂牌房源量 条件说明: 1.输入面积-5<=房源面积<=输入面积+5 2.上个月发布的房源 - * + * * @author yinjihuan * @date 2016年9月1日 * @param city @@ -4107,7 +4232,7 @@ public class PropertyController { /** * 小区周边一公里均价 - * + * * @author wenyan * @param model * @return @@ -4130,7 +4255,7 @@ public class PropertyController { /** * 相同小区相似户型租金范围查询接口 - * + * * @param model * @return */ @@ -4173,7 +4298,7 @@ public class PropertyController { /** * 周边小区相似户型租金范围查询接口 - * + * * @param model * @return */ @@ -4236,7 +4361,7 @@ public class PropertyController { /** * 室号查询接口 - * + * * @param model * @return */ @@ -4272,7 +4397,7 @@ public class PropertyController { /** * 楼栋楼层信息查询接口 - * + * * @param model * @return */ @@ -4297,7 +4422,7 @@ public class PropertyController { /** * 楼栋查询接口 - * + * * @param model * @return */ @@ -4342,7 +4467,7 @@ public class PropertyController { /** * 楼栋查询接口 - * + * * @param model * @return */ @@ -4795,7 +4920,7 @@ public class PropertyController { /** * 房产交易记录查询 - * + * * @param model * @return */ @@ -4855,7 +4980,7 @@ public class PropertyController { /** * 房产拍卖记录 - * + * * @param model * @return */ @@ -4916,7 +5041,7 @@ public class PropertyController { /** * 房产近期评估次数 - * + * * @param model * @return */ @@ -5412,6 +5537,36 @@ public class PropertyController { } } + public Object searchHouseLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + + if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber()) || StringUtils.isBlank(model.getRoomNumber())){ + mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号,室为必填项")); + return mav; + } + + List houseLoudongs = houseLoudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber(), model.getRoomNumber()); + mav.setModel(BasicModel.buildOKModel().setResult(houseLoudongs)); + return mav; + } + + + public Object searchLoudongAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + + if (StringUtils.isBlank(model.getId()) || StringUtils.isBlank(model.getHouseNumber())){ + mav.setModel(BasicModel.buildBadRequestModel("参数非法:楼盘编号,楼栋号为必填项")); + return mav; + } + + + List loudongs = loudongService.findAll(model.getId(), model.getHouseNumber(), model.getUnitNumber()); + mav.setModel(BasicModel.buildOKModel().setResult(loudongs)); + return mav; + } + public void setHouseLoudongService(HouseLoudongService houseLoudongService) { this.houseLoudongService = houseLoudongService; @@ -5601,4 +5756,16 @@ public class PropertyController { public void setRentDealPriceService(RentDealPriceService rentDealPriceService) { this.rentDealPriceService = rentDealPriceService; } + + public void setCommunityPoiService(CommunityPoiService communityPoiService) { + this.communityPoiService = communityPoiService; + } + + public void setEvaluateFactorService(EvaluateFactorService evaluateFactorService) { + this.evaluateFactorService = evaluateFactorService; + } + + public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) { + this.communityBasePriceService = communityBasePriceService; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java new file mode 100644 index 000000000..d23a312bb --- /dev/null +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/requestController.java @@ -0,0 +1,36 @@ +package com.fangjia.open.controllers; + +import com.fangjia.core.util.AnalyticsUtil; +import com.fangjia.open.interceptors.Authority; +import com.fangjia.open.interceptors.Role; +import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.RestfulModelAndView; + +import javax.servlet.http.HttpServletRequest; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; + +public class requestController { + + /** + * 获取请求的request和IP信息 + * @param request + * @return + */ + public Object requestAction(HttpServletRequest request) { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + String ip = AnalyticsUtil.getIpAddress(request); + Map map = new HashMap(); + Enumeration headerNames = request.getHeaderNames(); + while (headerNames.hasMoreElements()) { + String key = (String) headerNames.nextElement(); + String value = request.getHeader(key); + map.put(key, value); + } + map.put("ip", ip); + mav.setModel(BasicModel.buildOKModel().setResult(map)); + return mav; + } +} diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java index 4cc728b6a..826c55070 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/DistrictController.java @@ -80,7 +80,7 @@ public class DistrictController { // response.sendRedirect("/coming"); // return null; // } - if("b".equals(login_user_type)){ + if("b".equals(login_user_type) || "gb".equals(login_user_type)){ CacheUtils.putValue(CachePolicyEnum.LONG, UsersInfoService.class, weiXinId + "_city_"+ login_user_type , model.getCityName()); } Long userId = Long.parseLong(CookieUtils.getCookieValue(request,"login_user_id")); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/UserController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/UserController.java index 81419d5e4..e97bc10ae 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/UserController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/UserController.java @@ -80,7 +80,19 @@ public class UserController { } else { weiXinId = CookieUtils.getCookieValue(request,"we_chat_open_id"); } + + String cacheWeiXinId = CookieUtils.getCookieValue(request,"login_enterpriseuser_weixinid"); + try { + if(StringUtils.isNotBlank(weiXinId) && StringUtils.isNotBlank(cacheWeiXinId)){ + System.out.println("cookie weiXinId:"+weiXinId); + response.sendRedirect("reqCityChoicePage?type=b&code="+model.getCode()); + } + } catch (IOException e) { + e.printStackTrace(); + } + if(weiXinId == null && StringUtils.isNotBlank(model.getCode())) {//微信公众号渤海银行 + System.out.println("bh OPEN_ID 随机生成:" + weiXinId); weiXinId = DataConvertUtil.getRandomString2(16); } if(weiXinId != null && !weiXinId.equals("")){ @@ -115,33 +127,49 @@ public class UserController { if(weiXinId != null && !weiXinId.equals("")){ CookieUtils.setCookie("we_chat_open_id", weiXinId, "/", 60 * 60 * 24 *7, response); } - System.out.println("微信OPEN_ID:" + weiXinId); + System.out.println("reqCityChoicePage 微信OPEN_ID:" + weiXinId); //企业用户入口查询企业账户开通的城市 - if(type.equals("b")){ + if(type.equals("b") || type.equals("gb")){ //Boolean isLogin = usersInfoService.checkEnterpriseWeiXinUserIsLogin(weiXinId); String cacheWeiXinId = CookieUtils.getCookieValue(request,"login_enterpriseuser_weixinid"); - if(cacheWeiXinId != null) { - Long userId = Long.parseLong(CookieUtils.getCookieValue(request,"login_user_id")); + String loginUserIdstr = CookieUtils.getCookieValue(request,"login_user_id"); + if(cacheWeiXinId != null && StringUtils.isNotBlank(loginUserIdstr)) { + Long userId = Long.parseLong(loginUserIdstr); CookieUtils.setCookie("login_user_id", userId + "", "/", 60 * 60 * 24 *7, response); try { FjtUsers users = fjtUsersService.findById(userId); - List citys = userCityService.findSupportCityByUserId(userId); - if(citys == null || citys.isEmpty()){ - citys = Arrays.asList("上海"); - } + + if(users.getRole()==17 && type.equals("gb")){ + CookieUtils.removeCookie(request, response, "login_enterpriseuser_weixinid"); + response.sendRedirect("reqLoginPage"); + }else if(users.getRole()==12 && type.equals("b")){ + CookieUtils.removeCookie(request, response, "login_enterpriseuser_weixinid"); + response.sendRedirect("bhydLoginPage?code="+model.getCode()); + }else{ + List citys = userCityService.findSupportCityByUserId(userId); + if(citys == null || citys.isEmpty()){ + citys = Arrays.asList("上海"); + } // if(users!=null && users.getRole() == 12){ // citys = Arrays.asList("上海"); // }else{ // citys = userCityService.findSupportCityByUserId(userId); // } - model.setCitys(citys); + model.setCitys(citys); + } } catch (Exception e) { model.setCitys(new ArrayList()); } } else { + CookieUtils.removeCookie(request, response, "login_enterpriseuser_weixinid"); try { - response.sendRedirect("reqLoginPage"); +// if(type.equals("bh")){ +// response.sendRedirect("bhydLoginPage?code="+model.getCode()); +// }else{ + response.sendRedirect("reqLoginPage"); +// } + } catch (IOException e) { } } @@ -173,6 +201,20 @@ public class UserController { } catch (Exception e) { } } + + public Object bhloginOutAction(UserModel model,HttpServletRequest request, HttpServletResponse response, String city) { + try { + CookieUtils.removeCookie(request, response,"login_user_id"); + CookieUtils.removeCookie(request, response,"login_user_type"); + CookieUtils.removeCookie(request, response,"login_enterpriseuser_weixinid"); + response.sendRedirect("bhydLoginPage?code=bhydcode"); + } catch (Exception e) { + e.printStackTrace(); + } + return model; + } + + public Object aboutAction(UserModel model) { return model; @@ -285,7 +327,7 @@ public class UserController { return resultModel; } CookieUtils.setCookie("login_user_id", user.getId() + "", "/", 60 * 60 * 24 *7, response); - CookieUtils.setCookie("login_user_type", "b", "/", 60 * 60 * 24 *7, response); + CookieUtils.setCookie("login_user_type", "gb", "/", 60 * 60 * 24 *7, response); CookieUtils.setCookie("login_enterpriseuser_weixinid", weiXinId, "/", 60 * 60 * 24 *7, response); //CacheUtils.putValue(CachePolicyEnum.LONG, UsersInfoService.class, weiXinId + "_" + Message.SMS_CODE_TYPE_ENTER_LOGIN , user.getId()); resultModel.setResult(Message.RESULT_SUCCESS); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 5b70348d4..272979fca 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -49,6 +49,7 @@ import org.springframework.web.multipart.commons.CommonsMultipartFile; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.*; +import java.net.URL; import java.text.DecimalFormat; import java.text.SimpleDateFormat; import java.util.*; @@ -452,12 +453,33 @@ public class WeiXinEvaluationController { if(fjtUser!=null && fjtUser.getRole()==17) { System.out.println("渤海地址搜索 "+evaluation.getAddress()); List bhDists = BHBankUtils.getDistrictByBhbank(evaluation.getAddress()); - if(bhDists.size()<1){ + JSONObject o = null; + if(bhDists.size()==0){ bhDists = BHBankUtils.getDistrictByBhbank(district.getName()); } - Map bhMatchMap = new HashMap(); if(bhDists.size()==1){ - JSONObject o = bhDists.get(0); + o = bhDists.get(0); + }else if(bhDists.size()>1){ + int mod = Integer.MAX_VALUE; + for(JSONObject jo : bhDists){ + int bhprice = 0; + if (jo.get("price") == null || jo.get("price").equals("null")) { + bhprice = 0; + } else { + double price = Double.valueOf((String)jo.get("price")); + bhprice = (int)price; + } + + int bhmod = Math.abs(bhprice - avgPrice); + if(bhmod bhMatchMap = new HashMap(); + if(o != null){ + String bhDistName = (String)o.get("name"); String regionCode = String.valueOf((Long)o.get("country")); String bhRegion = BHBankUtils.regionCodeMap.get(regionCode); @@ -982,13 +1004,14 @@ public class WeiXinEvaluationController { if(StringUtils.isNotBlank(model.getEvaluationId()) && StringUtils.isNotBlank(model.getEmail())){ UserTrades userTrade = userTradesService.findUserTradesByEvaluationId(model.getEvaluationId()); + + FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId())); if(userTrade != null){ userTrade.setSend(1); userTrade.setSendEmail(model.getEmail()); userTradesService.update(userTrade); }else{ - FjtEvaluation evaluation = fjtEvaluationService.findEvaluationById(Long.valueOf(model.getEvaluationId())); - + userTrade = new UserTrades(); userTrade.setUserId(evaluation.getUserId()); userTrade.setCity(evaluation.getCity()); @@ -1008,20 +1031,44 @@ public class WeiXinEvaluationController { String reporNum = "147698"+ String.valueOf(evaluation.getId()); evaluation.setReportNum(reporNum); + evaluation.setDownPdfDate(new Date()); fjtEvaluationService.updateEvaluation(evaluation); try { String smsContent = "【房价网】微信询价报告发送 用户邮箱:"+model.getEmail(); smsService.sendDingMsgByRobot(DD_ID, smsContent); - + } catch (SmsException e) { e.printStackTrace(); } } - - + sendToEmail(model.getFjtUrlKey(), model.getEvaluationId(), model.getEmail()); - + FjtUsers users = fjtUsersService.findById(evaluation.getUserId()); + if(users!=null && users.getRole()==17){ + + String converFileName = DateUtils.format(new Date(), "yyyy-MM-dd") +"_"+evaluation.getReportNum()+".pdf"; + String url = model.getFjtUrlKey() + "/evaluation/exportPdf?evaluationId="+model.getEvaluationId()+"&bkoffDownload=true&userId="+users.getId(); + String fontUrl = ActionContext.getContext().getServletContext().getRealPath("/") + "files" + File.separator + model.getEvaluationId() + File.separator; + + System.out.println(fontUrl); + try { + createFile(fontUrl+converFileName); + download(url, fontUrl, converFileName); + List uploadfiles = new ArrayList(); + uploadfiles.add(new File(fontUrl+converFileName)); + if(uploadfiles.size()>0){ + BohaiEvaluationAudit boAudit = bohaiEvaluationAuditService.findByBohaiEvaluationAuditId(evaluation.getId()); + if(boAudit != null){ + UsersDetail usersDetail = usersDetailService.findById(users.getId()); + BHBankUtils.inquireUploadFiles2(usersDetail.getName(),boAudit.getQueryId(), 1, uploadfiles); + } + } + } catch (IOException e) { + //处理异常信息 + System.out.println(e.getMessage()); + } + } } return jsonMmodel; } @@ -1307,8 +1354,45 @@ public class WeiXinEvaluationController { } } + public static void download(String url, String saveDir, String fileName) { + + BufferedOutputStream bos = null; + InputStream is = null; + try { + byte[] buff = new byte[8192]; + is = new URL(url).openStream(); + File file = new File(saveDir, fileName); + file.getParentFile().mkdirs(); + bos = new BufferedOutputStream(new FileOutputStream(file)); + int count = 0; + while ( (count = is.read(buff)) != -1) { + bos.write(buff, 0, count); + } + } + catch (IOException e) { + e.printStackTrace(); + } + finally { + if (is != null) { + try { + is.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + if (bos != null) { + try { + bos.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + + } //将InputStream类型转化为File类型 - private static void inputStreamToFile(InputStream ins, File file) { + private static void inputStreamToFile(InputStream ins, File file) { //将输出流转化为文件即可 FileOutputStream os = null; try { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java index 1ab67359e..d938d3a8b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/dto/DistrictInfoDto.java @@ -99,6 +99,7 @@ public class DistrictInfoDto { private String salesPhoneNumber; private Integer buildingCount; private Integer finishYear; + private String township; public String getDistrict() { @@ -818,4 +819,12 @@ public class DistrictInfoDto { public void setFinishYear(Integer finishYear) { this.finishYear = finishYear; } + + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java b/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java index e16664d16..37805ef21 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/filters/AnalyticsFilter.java @@ -75,8 +75,8 @@ public class AnalyticsFilter implements Filter { if ("1".equals(request.getParameter("__refresh_cache"))) { threadLocalData.put(ThreadLocalKey.DisableCache, "1"); } - createUID(request, response); - createSID(request, response); + //createUID(request, response); + //createSID(request, response); response.addHeader("Access-Control-Allow-Origin", "*"); response.addHeader("Access-Control-Allow-Methods", "*"); chain.doFilter(request, response); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java index a4d47a4e6..65092281a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AuctionModel.java @@ -8,6 +8,7 @@ public class AuctionModel extends FangjiaApiModel{ private String city; private String district; private String address; + private String name; private Date minStartTime; private Date maxStartTime; private Integer minStartPrice; @@ -17,6 +18,7 @@ public class AuctionModel extends FangjiaApiModel{ private Integer propertyType; private Integer status; private Integer limit = 10; + private String startDate; public String getId() { return id; @@ -121,4 +123,20 @@ public class AuctionModel extends FangjiaApiModel{ public void setStatus(Integer status) { this.status = status; } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java index 0d51d0c46..35620322f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/DealPriceModel.java @@ -1,6 +1,9 @@ package com.fangjia.open.model; +import java.util.Date; + public class DealPriceModel extends FangjiaApiModel{ + private String id; private String city; @@ -34,9 +37,13 @@ public class DealPriceModel extends FangjiaApiModel{ private Double size; private Integer max; + private Integer limit = 10; private String decoration; private boolean convert; + + private String startDate; + private String endDate; public String getCity() { @@ -198,4 +205,36 @@ public class DealPriceModel extends FangjiaApiModel{ public void setConvert(boolean convert) { this.convert = convert; } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java index d5b8f42ed..71130e7bb 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/EvaluationModel.java @@ -246,6 +246,42 @@ public class EvaluationModel extends FangjiaApiModel { private Date buildDate; private String sid;//微信小程序中的sid private String code;//微信小程序中的code + private String id; + private String startDate; + private String endDate; + private Integer limit = 10; + + public Integer getLimit() { + return limit; + } + + public void setLimit(Integer limit) { + this.limit = limit; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getStartDate() { + return startDate; + } + + public void setStartDate(String startDate) { + this.startDate = startDate; + } + + public String getEndDate() { + return endDate; + } + + public void setEndDate(String endDate) { + this.endDate = endDate; + } public String getRegion() { return region; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java index d9f4812d8..63a85c4b3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/HouseSearchModel.java @@ -4,6 +4,7 @@ import com.fangjia.service.datastore.bean.HouseQueryBean; import com.fangjia.service.query.QueryBean; public class HouseSearchModel extends SearchModel { + private String id; private String kind; private String estateType; private String buildingType; @@ -116,7 +117,12 @@ public class HouseSearchModel extends SearchModel { public void setShowDetail(Boolean showDetail) { this.showDetail = showDetail; } - - - + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java index 5953fc3a3..859658742 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java @@ -46,7 +46,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel { * 楼号 */ private String unitNumber; + /** + * 新楼号 + */ + private String houseNumber; + private String id; /** * 室号 */ @@ -905,4 +910,20 @@ public class PropertyEvaluationModel extends FangjiaApiModel { public void setAvgPrice(Integer avgPrice) { this.avgPrice = avgPrice; } + + public String getHouseNumber() { + return houseNumber; + } + + public void setHouseNumber(String houseNumber) { + this.houseNumber = houseNumber; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java index 813f42f6c..64fb38772 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/SearchModel.java @@ -16,6 +16,8 @@ public abstract class SearchModel extends FangjiaApiModel { protected Integer totalPriceEnd; protected Integer avgPriceStart; protected Integer avgPriceEnd; + protected Integer startTime; + protected Integer endTime; protected Integer areaStart; protected Integer areaEnd; protected Integer room; @@ -46,6 +48,7 @@ public abstract class SearchModel extends FangjiaApiModel { private Double area; protected Integer dataPeriod;//查询天数 + private Integer distCount; /** * 获取房源数量类型 sell:只取得二手房数量 rent:只取得出租房数量 其他:不获取数量 @@ -358,4 +361,28 @@ public abstract class SearchModel extends FangjiaApiModel { public void setTotalPrice(Integer totalPrice) { this.totalPrice = totalPrice; } + + public Integer getStartTime() { + return startTime; + } + + public void setStartTime(Integer startTime) { + this.startTime = startTime; + } + + public Integer getEndTime() { + return endTime; + } + + public void setEndTime(Integer endTime) { + this.endTime = endTime; + } + + public Integer getDistCount() { + return (null != distCount && 0 != distCount) ? distCount : 20; + } + + public void setDistCount(Integer distCount) { + this.distCount = distCount; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index 3229b9845..2c115f3b4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -2,27 +2,15 @@ package com.fangjia.open.util; import com.fangjia.core.util.HttpRequest; import com.fangjia.core.util.MD5Util; -import com.fangjia.core.util.MultipartUtility; import com.fangjia.dal.sql.model.FjtEvaluation; import com.fangjia.dal.sql.model.UsersDetail; import com.fangjia.open.model.weixin.EvaluationModel; -import com.mashape.unirest.http.HttpResponse; -import com.mashape.unirest.http.Unirest; -import com.mashape.unirest.request.HttpRequestWithBody; -import com.yeepay.shade.org.apache.http.entity.mime.MultipartEntityBuilder; import net.sf.json.JSONArray; import net.sf.json.JSONObject; import okhttp3.*; -import org.apache.commons.collections.map.MultiValueMap; import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.springframework.http.ResponseEntity; import java.io.File; -import java.io.FileInputStream; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.*; @@ -58,8 +46,11 @@ public class BHBankUtils { }}; public static List getDistrictByBhbank(String address){ - List list = new ArrayList(); + try { + +// Thread.sleep(20*1000); + Map params = new HashMap(); params.put("keyword",address); @@ -73,7 +64,6 @@ public class BHBankUtils { headers.put("nonce", nonce.toString()); HttpRequest request = new HttpRequest(); String content = null; - try { content = request.get(uri +"/searchAddr/bh/", params, headers, "utf-8"); System.out.println("/searchAddr/bh/ :"+content); if(StringUtils.isNotBlank(content)){ @@ -130,8 +120,19 @@ public class BHBankUtils { params.put("houseNo",ev.getRoomNum()); } if(ev.getHouseArea()>0){ - params.put("groundArea", String.valueOf(ev.getHouseArea())); + if(ev.getBasementArea()!=null && ev.getBasementArea()>0){ + double area = ev.getHouseArea() - ev.getBasementArea(); + if(area>0){ + params.put("groundArea", String.valueOf(area)); + } + }else{ + params.put("groundArea", String.valueOf(ev.getHouseArea())); + } } + if(ev.getBasementArea()!=null && ev.getBasementArea()>0){ + params.put("undergroundArea", String.valueOf(ev.getBasementArea())); + } + params.put("orientName", "南"); params.put("buildYear", ev.getFinishDateStr()); fjwTotalPrice= (int)(fjwTotalPrice/10000) * 10000; @@ -440,7 +441,7 @@ public class BHBankUtils { Map params = new HashMap(); params.put("queryId",queryId); if(StringUtils.isNotBlank(reportMsg)){ - params.put("reportMsg",reportMsg); + params.put("reportMsg",reportMsg.replace(" ","")); } diff --git a/projects/fangjia-open/src/main/resources/fangjia-open.properties b/projects/fangjia-open/src/main/resources/fangjia-open.properties index aaf6af5ca..c7d4d526c 100644 --- a/projects/fangjia-open/src/main/resources/fangjia-open.properties +++ b/projects/fangjia-open/src/main/resources/fangjia-open.properties @@ -100,4 +100,5 @@ fangjia.jms.email.queue=email.queue fangjia.redis_nodes=192.168.0.79:6379,192.168.0.79:6380,192.168.0.79:6381,192.168.0.81:6382,192.168.0.81:6383,192.168.0.81:6384 -fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2 \ No newline at end of file +fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2 +fangjia.default.city = \u91CD\u5E86\u5E02 \ No newline at end of file diff --git a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml index fb8d631db..c17530d88 100644 --- a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml +++ b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml @@ -1,8 +1,8 @@ - - + + @@ -33,5 +33,15 @@ /property/avgPrice + + + /address/standard + + + + + /address/fiveDivisionStructure + + \ No newline at end of file diff --git a/projects/fangjia-open/src/main/webapp/district/weXinList.ftl b/projects/fangjia-open/src/main/webapp/district/weXinList.ftl index 3e07b5ef4..46ef7c552 100644 --- a/projects/fangjia-open/src/main/webapp/district/weXinList.ftl +++ b/projects/fangjia-open/src/main/webapp/district/weXinList.ftl @@ -26,7 +26,7 @@
- + <#if !(cityName??) || cityName == ""> <#else> @@ -48,7 +48,7 @@
    - <#if type?? && type == 'b'> + <#if type?? && (type == 'b' || type == 'gb')>
    - <#if type?? && type == 'b' && (citys??&&citys?size>0)> + <#if type?? && (type == 'b' || type == 'gb') && (citys??&&citys?size>0)>
    <#list citys as bo>

    ${bo}

    @@ -54,7 +54,7 @@
    - <#if type?? && type == 'c' || type == 'b' && (citys??&&citys?size=0)> + <#if type?? && type == 'c' || (type == 'b' || type == 'gb') && (citys??&&citys?size=0)>
    褰撳墠瀹氫綅鍩庡競
    diff --git a/projects/fangjia-open/src/main/webapp/user/reqLoginPage.ftl b/projects/fangjia-open/src/main/webapp/user/reqLoginPage.ftl index 621633126..f9d2c03e3 100644 --- a/projects/fangjia-open/src/main/webapp/user/reqLoginPage.ftl +++ b/projects/fangjia-open/src/main/webapp/user/reqLoginPage.ftl @@ -84,7 +84,7 @@ if(re.result == "fail"){ $.toast(re.msg); } else { - window.location.href = "reqCityChoicePage?type=b"; + window.location.href = "reqCityChoicePage?type=gb"; } },error:function (data, status) { $.toast("鍙戦�佸け璐ワ紝璇烽噸璇�"); diff --git a/projects/fangjia-open/src/main/webapp/user/setup.ftl b/projects/fangjia-open/src/main/webapp/user/setup.ftl index a173193de..797f87a6c 100644 --- a/projects/fangjia-open/src/main/webapp/user/setup.ftl +++ b/projects/fangjia-open/src/main/webapp/user/setup.ftl @@ -67,7 +67,7 @@
    +
  • +
    +
    + *寤虹瓚闈㈢Н涓惈鏈夊湴涓嬮檮灞為潰绉� +
    +
    +
  • diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index e70200bda..165012ff6 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -44,7 +44,7 @@ <#if role?? && role==17 && loanInfoMap??>

    鎵�灞炲垎绫伙細${loanInfoMap.type!}绫�

    鍙捶鎴愭暟锛�${loanInfoMap.loan_ratio!}鎴�

    -

    鍙捶閲戦锛�${(fjtEvaluation.totalPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

    +

    鍙捶閲戦锛�${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

    璐锋鍒╃巼锛�4.5%

    <#else> @@ -88,7 +88,7 @@
    鏈熸湜浠凤細 -  涓囧厓 + disabled> 涓囧厓 鎻愪氦 鍒锋柊
    diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee26c1830..9469b6d02 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -1,5 +1,5 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 com.fangjia fangjia-script @@ -62,7 +62,7 @@ - org.eclipse.m2e @@ -141,11 +141,11 @@ activemq-pool 5.5.0 - - cn.emay - emayclient - 4.3 - + + cn.emay + emayclient + 4.3 + org.apache.activemq activemq-core @@ -366,18 +366,20 @@ org.apache.poi poi + 3.10-FINAL org.apache.poi poi-ooxml + 3.10-FINAL - - - xml-apis - xml-apis - - + + + xml-apis + xml-apis + + " + name) + name = name.replaceAll("路|-", '') + println(name) + } + if (name ==~ /.{2,}璺�.{3,}$/ && !(name ==~ /.{2,}璺�.+鍙�/) && !(name ==~ /.{2,}璺�.+寮�/) && !(name ==~ /.*灏忓尯$/)) { + name = name.replaceAll("^.{2,}璺�", "") + } + return name + } + + + static Object parse(String content, Map resolvers) { + if (content != null) { + content = content.replaceAll("\\s", "") + } + for (def resolver : resolvers.entrySet()) { + String match = match(content, resolver.getKey()) + if (StringUtils.isNotBlank(match)) { + match = match.replace("涓�","1").replace("浜�","2").replace("涓�","3").replace("鍥�","4").replace("浜�","5").replace("涓�","2") + return resolver.getValue()(match) + } + } + return null + } + + /** + * 鍖归厤绗﹀悎鏉′欢鐨勫瓧绗︿覆 + * + * @param str + * @param pattern + * @return + */ + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + static String parseFitment(String fitmentDesc) { + if (fitmentDesc == null) { + return null + } + fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "") + if (fitmentDesc.equals("闈炴瘺鍧�")) { + return null + } + Iterator>> it = FITMENTS_TYPES_MAPS + .entrySet().iterator() + Map.Entry> entry + while (it.hasNext()) { + entry = it.next() + if (fitmentDesc.contains(entry.getKey())) { + return entry.getKey() + } + for (String fitmentKey : entry.getValue()) { + if (fitmentDesc.contains(fitmentKey)) { + return entry.getKey() + } + } + } + return null + + } + + static String parseDirection(String directionDesc) { + if (directionDesc == null) { + return null + } + directionDesc = directionDesc.replaceAll("(\\(|锛坾銆妡\"|鈥渱銆恷銆�).+?(\\)|銆媩锛墊\"|鈥潀銆憒銆�)", "#_#") + StringBuffer sb = new StringBuffer() + for (int i = 0; i < directionDesc.length(); i++) { + String c = directionDesc.substring(i, i + 1) + if (DIRECTIONS.contains(c)) { + if (directionDesc.substring(i).startsWith(c + "#_#")) { + sb.append(c + " ") + } else { + sb.append(c) + } + } + } + for (String d : DIRECTIONS) { + if (sb.toString().startsWith(d)) { + return d + } + } + for (String d : DIRECTIONS) { + if (sb.toString().contains(d)) { + return d + } + } + return null + + } + + + static String parsePrice(String priceStr) { + def pattern =Pattern.compile("\\d+(\\.\\d+)?涓�") + String match = match(priceStr, pattern) + } + + static void main(String[] args) { + println parsePrice("椤剁娆e洯瑗胯嫅2瀹�1鍘�1鍗�492.3涓�") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy new file mode 100644 index 000000000..845ab47d0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/crawl/ZhugeFangDistCrawl.groovy @@ -0,0 +1,167 @@ +package com.fangjia.script.gj.crawl + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.thirdPriceCrawl.Utils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class ZhugeFangDistCrawl { + + public static final SHORT_DATE = Utils.getLastMonth(); + def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new ZhugeFangDistCrawl(); + task.run(); + } + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + def url = "https://fz.esfxiaoqu.zhuge.com/fuqing/" + WebDriver driver = new ChromeDriver() + driver.manage().deleteAllCookies() + getRegions(driver, url) + } + + def getRegions(driver, url) { + for(def i=1;i<=11;i++){ + def newUrl = url+"page${i}/" + println newUrl + def doc = getDocument(newUrl, driver) + def total = parseList(doc) + +// def next = JQueryUtils.select("a:contains(下一页)", doc) +// if(next){ +// driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() +// } + + } + + } + + def parseList(doc){ + def city= "福州" + def els = JQueryUtils.select("#listTableBox li", doc) + if (els) { + for(def e : els){ + def name = e.select(".house-name").text() + def str = e.select(".house-adr").text() + def address = null + if(str){ + if(str.contains("]")){ + address = str.substring(str.indexOf("]")+1).trim() + }else{ + address = str + } + } + def regionStr = null + if(str && str.contains("]") && str.contains("[")){ + regionStr = str.substring(str.indexOf("[")+1, str.indexOf("]")) + } + + def region = null + def block = null + if(regionStr){ + def arrs = regionStr.split("-") + if(arrs && arrs.size()>0){ + if(arrs.size()==2){ + block = arrs[1].trim() + } + if(arrs.size()>=1){ + region = arrs[0].trim() + } + + } + } + def links = e.select(".house-name a") + def href = null + if(links){ + href = links[0].attr("href") + } + def priceStr = e.select(".price-number").text() + def price = null + if(priceStr && !priceStr.contains("暂无")){ + if(priceStr.contains("元")){ + priceStr = priceStr.substring(0, priceStr.indexOf("元")).trim() + } + if(priceStr && priceStr.isNumber()){ + price = priceStr.toInteger() + } + } + def obj = findByParams(city, region, name) + if(obj){ + if(price){ + DBObject udata2 = new BasicDBObject() + udata2.put("avg_price", price) + udata2.put("m_date", new Date()) + println "[upeate] ${region} "+name + thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2)); + } + }else{ + def udata = new BasicDBObject() + udata.put("city", city) + udata.put("region", region) + udata.put("block", block) + udata.put("url", href) + udata.put("address", address) + udata.put("name", name) + udata.put("avg_price", price) + udata.put("s_date", SHORT_DATE) + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + udata.put("source", "诸葛找房") + thirdPartyColl.save(udata) + println "[save] ${region} "+name+" "+priceStr + } + } + } + return els.size() + } + + + def findByParams(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("source", "诸葛找房") + q.put("s_date", SHORT_DATE) + return thirdPartyColl.findOne(q); + } + + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + Thread.sleep(10000) + def els = JQueryUtils.select("#listTableBox li", doc) + if(els && els.size()>0){ + stop = true; + }else{ + error++ + driver.manage().deleteAllCookies() + println "[${error}] error: "+url + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=20){ + stop = true; + } + } + return doc + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy index 92e8f1555..7730b81cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/AddDistrictFieldValue.groovy @@ -55,7 +55,7 @@ class AddDistrictFieldValue { // } // }) -// def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys; + def citys = ["重庆"]//CcbCommonUtils.citys; //第三方均价数据匹配 // for(def city : citys){ // def rets = findAllDataByCity(city) @@ -63,11 +63,11 @@ class AddDistrictFieldValue { // } //第三方均价数据填补 -// for(def city : citys){ -//// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) -// def rMap = getMatchDistrict(city) -// createResultValue(dists, rMap) -// } + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"]) + def rMap = getMatchDistrict(city) + createResultValue(dists, rMap) + } // for(def city : citys){ // def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"]) @@ -103,52 +103,52 @@ class AddDistrictFieldValue { // } - def citys = ["上海"] - def updateIndex = 0 - def rets = [] - rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"] - for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"]) - def total = dists.size() - - for(def dist : dists){ - if(dist.household_count){ - def ret = [] - ret << dist._id.toString() - ret << dist.city - ret << dist.region - ret << dist.name - ret << dist.household_count - ret << dist.loudong_count - def details = findDistrictDetail(dist) - if(details){ - for(def d : details){ - def region = RegionData.formatRegion(city, d.region) - if(d.region == region){ - if(dist.household_count && dist.household_count==d.house_count){ - ret << d.loudong_count - ret << "链家" -// updateIndex++ -// def udata = new BasicDBObject() -// udata.put("household_count", d.house_count.toInteger()) -// udata.put("m_date", new Date()) -// def q = new BasicDBObject() -// q.put("_id", dist._id) -// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - } - } - } - }else{ - ret << "" - ret << "" - } - - rets << ret - } - } - ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据") - } +// def citys = ["上海"] +// def updateIndex = 0 +// def rets = [] +// rets << ["编号", "城市", "区域", "名称","户数", "房价网楼栋数","建行楼栋数","来源"] +// for(def city : citys){ +// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "household_count","loudong_count", "alias"]) +// def total = dists.size() +// +// for(def dist : dists){ +// if(dist.household_count){ +// def ret = [] +// ret << dist._id.toString() +// ret << dist.city +// ret << dist.region +// ret << dist.name +// ret << dist.household_count +// ret << dist.loudong_count +// def details = findDistrictDetail(dist) +// if(details){ +// for(def d : details){ +// def region = RegionData.formatRegion(city, d.region) +// if(d.region == region){ +// if(dist.household_count && dist.household_count==d.house_count){ +// ret << d.loudong_count +// ret << "链家" +//// updateIndex++ +//// def udata = new BasicDBObject() +//// udata.put("household_count", d.house_count.toInteger()) +//// udata.put("m_date", new Date()) +//// def q = new BasicDBObject() +//// q.put("_id", dist._id) +//// println updateIndex +"/${total} "+d.city +" "+d.name+" old-vale:"+dist.household_count+" new-value:"+d.house_count +//// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// } +// } +// } +// }else{ +// ret << "" +// ret << "" +// } +// +// rets << ret +// } +// } +// ExcelUtils.write("d:\\数据\\上海楼盘户数.xlsx", rets, "数据") +// } } @@ -303,6 +303,14 @@ class AddDistrictFieldValue { def createResultValue(dists, rMap){ def updateIndex = 0 def total = dists.size() + def addressIndex = 0 + def allAreaIndex = 0 + def buildAreaIndex = 0 + def virescence_percent_index = 0 + def cubage_percent_index = 0 + def finish_building_date_index = 0 + def household_count_index = 0 + def car_site_count_index = 0 for(def d : dists){ def key = d.city +"#"+d.region+"#"+d.name def sourceMap = rMap.get(key) @@ -311,12 +319,14 @@ class AddDistrictFieldValue { if(!d.address){ def lastValue = getValuesByField(sourceMap,"address", d.estate_type2) if(lastValue){ + addressIndex++ udata.put("address", lastValue) } } if(!d.all_area){ def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2) if(lastValue){ + allAreaIndex++ udata.put("all_area", lastValue) } } @@ -324,6 +334,7 @@ class AddDistrictFieldValue { if(!d.build_area){ def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2) if(lastValue){ + buildAreaIndex++ udata.put("build_area", lastValue) } } @@ -331,6 +342,7 @@ class AddDistrictFieldValue { if(!d.virescence_percent){ def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2) if(lastValue){ + virescence_percent_index ++ udata.put("virescence_percent", lastValue) } } @@ -338,6 +350,7 @@ class AddDistrictFieldValue { if(!d.cubage_percent){ def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2) if(lastValue){ + cubage_percent_index ++ udata.put("cubage_percent", lastValue) } } @@ -345,6 +358,7 @@ class AddDistrictFieldValue { if(!d.finish_building_date){ def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2) if(lastValue){ + finish_building_date_index ++ udata.put("finish_building_date", lastValue) } } @@ -352,6 +366,7 @@ class AddDistrictFieldValue { if(!d.household_count){ def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2) if(lastValue){ + household_count_index ++ udata.put("household_count", lastValue) } } @@ -359,6 +374,7 @@ class AddDistrictFieldValue { if(!d.car_site_count){ def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2) if(lastValue){ + car_site_count_index ++ udata.put("car_site_count", lastValue) } } @@ -373,6 +389,15 @@ class AddDistrictFieldValue { } } } + + println "addressIndex "+ addressIndex + println "allAreaIndex "+ allAreaIndex + println "buildAreaIndex "+ buildAreaIndex + println "virescence_percent_index "+ virescence_percent_index + println "cubage_percent_index "+ cubage_percent_index + println "finish_building_date_index "+ finish_building_date_index + println "household_count_index "+ household_count_index + println "car_site_count_index "+ car_site_count_index } def getValuesByField(sourceMap, field, estateType){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy new file mode 100644 index 000000000..13b8fdde9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy @@ -0,0 +1,82 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import org.apache.xmlbeans.impl.regex.Match + +import java.util.regex.Pattern + +class TempMatchDistrict { + static main(args) { + def task = new TempMatchDistrict() +// task.run() +// task.regionConver() + task.stat() + } + + def stat(){ + def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] + def total = 0 + for(def city : citys){ + def count = CommonDatautils.findAllDistrictCount("district", city) + total = total + count + println city +" "+count + } + println "共:"+total + } + + def regionConver(){ + //,"北京" + def files = ["广州","杭州","合肥","上海","深圳","石家庄","苏州","天津","武汉","西安","重庆"] + for(def f : files){ + ExcelUtils.multiThreadingModify("D:\\数据\\其它\\均价数据\\"+f+".xlsx",{d-> + def orgName = CommonDatautils.getOrgName(f, d['M']) + if(orgName){ + d['M'] = orgName + } + }) + } + } + def run(){ + def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据" + def files = ["杭州.xlsx"] + for(def f : files){ + def filePath = path+"\\"+f; + match(filePath); + } + } + def match(filePath){ + ExcelUtils.multiThreadingModify(filePath,{d-> + def city = d['B'].replace("市","") +// def newRegion = RegionData.formatRegion(city, d['C']) +// if(newRegion){ + def rMap = SeaweedDataMatchProcess.doMatch(city, d['C'], d['D'], null); + if(rMap){ + def dist = CommonDatautils.queryDistrictById(rMap.id) + if(dist){ + d['N'] = dist._id.toString() + d['O'] = CommonDatautils.getOrgName(city, dist.region) + d['P'] = dist.name + d['Q'] = dist.lng2/10000000000.0 + d['R'] = dist.lat2/10000000000.0 + d['S'] = getYear(dist.finish_building_date) + } + } +// } + }) + } + + + def getYear(finishBuildingDate){ + if(finishBuildingDate){ + Pattern pat3 = Pattern.compile("\\d{4}"); + def matcher3 = pat3.matcher(finishBuildingDate); + if(matcher3.find()){ + return matcher3.group() + } + } + return null + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index ac634e9e9..12d2f77b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -10,7 +10,7 @@ class FangjiaClientUtils { public static def getMapData(def city, def region, def name, def address, def types, def mapCode){ JSONObject jsonObj = null try { - def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode //def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode if(address){ url = url + "&address="+address @@ -38,8 +38,8 @@ class FangjiaClientUtils { } public static def sendDistrictPriceCalculate(def city, def region, def name, def count){ - def url = "http://192.168.0.19:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" -// def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" +// def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" + def url = "http://192.168.10.216:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" def content = null; try{ content = new HttpRequest().newGet(url, null, "utf-8") @@ -83,6 +83,23 @@ class FangjiaClientUtils { return jsonObj; } + public static def arroundPrice(city,address){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + // def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.100:38080/FangJiaDataService/evaluate?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&maxNum=10&radius=1000&size=100" +// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + public static Map JsonToMap(JSONObject json) { Map columnValMap = new HashMap(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy index 71657eea3..cbee24228 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy @@ -172,7 +172,8 @@ class NameProcess { if((name.endsWith("村") || name.endsWith("街"))&& name.size()<=3){ return null } - if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("组") || (name.endsWith("阁") && name.length()<=3) || name.endsWith("条")){ + //|| name.endsWith("组") + if(name.endsWith("电梯")||name.endsWith("大道")||name.endsWith("厂") || name.endsWith("条")){ return null } if(name.contains("、")||name.contains("|")||name.contains(".")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 9219fc438..319e44ef4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -29,6 +29,7 @@ class SeaweedDataMatchProcess { rMap.lat = obj.lat/10000000000.0 rMap.lng = obj.lng/10000000000.0 rMap.id = obj._id.toString() + rMap.address = obj.address } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy new file mode 100644 index 000000000..5c87fa8cb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy @@ -0,0 +1,340 @@ +package com.fangjia.script.gj.district + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import com.fangjia.core.util.BaiDuApiUtils + +import java.sql.ResultSet + +class ZsNoExistsToDB { + + public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + public static EsDataSource dataSource = new EsDataSource() + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + + static main(args) { + def task = new ZsNoExistsToDB() + task.run() +// task.addDistrictToDB() + } + + def addDistrictToDB(){ + def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1) + def keys = [] + def addIndex = 0 + for(def d : datas){ + if(d['J'] && d['J']=="新增"){ + def city = d['A'] + def region = d['B'] + def name = d['F'].toString().replace("·","") + def address = d['G'] + def dist = CommonDatautils.findDistrict(city,region,name,"district") + if(!dist){ + dist = CommonDatautils.findDistrict(city,region,d['C'],"district") + } + if(!dist){ + keys << city+"##"+region+"##"+name + def alias = [] + def str = d['H'] + if(str){ + def arrs = str.split(",") + for(def s : arrs){ + if(!alias.contains(s)){ + alias << s + } + } + } + + if(address){ + address = AddressUtil.formatAddress(city, address) + } + def lng = d['M'] + def lat = d['L'] + def lng2 = null + def lat2 = null + long lng_new = 0 + long lat_new = 0 + long lng2_new = 0 + long lat2_new = 0 + if(lng){ + lng = lng.toString().toDouble() + lat = lat.toString().toDouble() + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + lng2 = baidu.x + lat2 = baidu.y + lng_new = lng *10000000000L + lat_new = lat *10000000000L + if(lng2){ + lng2_new = lng2 *10000000000L + lat2_new = lat2 *10000000000L + } + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + } + + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",region) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", address) + if(lng_new){ + data.put("lat",lat_new) + data.put("lng",lng_new) + data.put("lat2",lat2_new) + data.put("lng2",lng2_new) + } + if(alias){ + data.put("alias",alias) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + DbUtils.seaweed.save(data) + FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12); + addIndex++ + println "[新增] "+addIndex +" "+ data + + }else{ + println dist.city +" "+dist.region+" "+dist.name+" 小区已存在" + } + } + } + } + + def run(){ + def rets = findOpenLog2() + // ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据") + println rets.size() + +// matchMap2() + } + + def matchMap2() { + ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d -> + if(!d['E']){ + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject(d['A'],d['B'],d['C'], rets); + + def locationMap = [:] + if (!map || !map.lat) { + rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); + map = findStandardObject(d['A'],d['B'],d['C'], rets); + } + if(map){ + println map + if(map.adname){ + d['E'] = map.adname + }else if(map.region){ + d['E'] = map.region + } + d['F'] = map.name + d['G'] = map.address + d['K'] = map.estateType + d['L'] = map.lat + d['M'] = map.lng + d['N'] = map.lat2 + d['O'] = map.lng2 + d['P'] = map.source + d['Q'] = RegionData.formatRegion(d['A'], d['E']) + } + } + }) + } + + def findStandardObject(def city, def region, def name, def rets) { + def map; + if (rets) { + if (rets.size() >= 1) { + for (def mapObj : rets) { +// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){ + mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address); + map = mapObj; + break; + }else if(mapObj.address && mapObj.address.contains(name)){ + map = mapObj; + break; + }else if(mapObj.name && name){ + double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name); + if(score1){ + map = mapObj; + break; + } + } +// } + } + } + } + return map; + } + + + def matchMap(){ + ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data-> + def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德") + if(!gdData){ + gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德") + } + if(gdData){ + data['E'] = gdData.district + data['F'] = gdData.name + data['G'] = gdData.address + data['H'] = gdData.type + data['I'] = gdData.lng + data['J'] = gdData.lat + data['K'] = gdData.score + } + + def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度") + if(!bdData){ + bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度") + } + if(bdData){ + data['L'] = bdData.district + data['M'] = bdData.name + data['N'] = bdData.address + data['O'] = bdData.type + data['P'] = bdData.lng + data['Q'] = bdData.lat + data['R'] = bdData.score + } + },0) + } + + + def findOpenLog2(){ + def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18") + QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate) + def rets = [] + rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"] + def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def index = 0 + def count = 0 + def keys = [] + for(def d : datas){ + if(d.code == 400){ + index++ + def msg = d.msg + if(msg.contains("该楼盘不存在")){ + def arrs = msg.split(" ") + if(arrs){ + def rMap = [:] + for(def s : arrs){ + if(s.contains(":")){ + def ars = s.split(":") + if(ars.size()==2){ + rMap.put(ars[0], ars[1]) + }else{ + rMap.put(ars[0], null) + } + } + } + if(rMap){ + count++ + def key = rMap.city+"#"+rMap.region+"#"+rMap.name + if(!keys.contains(key)){ + keys << key + def params = d.parameter_map + def address = null + if(params && params.address){ + address = params.address + } + def r = [rMap.city, rMap.region, rMap.name, address] + if(!rets.contains(r)){ + rets << r + } + } + } + } + }else{ + println msg + } + } + } + println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size() + return rets; + } + + def findOpenLog(){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1") + sqlBuffer.append(" and user_id = '1007569'") + sqlBuffer.append(" and request_uri = '/czbank/evaluation'") + sqlBuffer.append(" and c_date >= '2022-01-01'") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def rets = [] + rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"] + def index = 0 + def count = 0 + while (rs.next()){ + def code = rs.getInt("code") + if(code == 400){ + index++ + def msg = rs.getString("msg") + if(msg && msg.contains("该楼盘不存在")){ + def arrs = msg.split(" ") + if(arrs){ + def rMap = [:] + for(def s : arrs){ + if(s.contains(":")){ + def ars = s.split(":") + if(ars.size()==2){ + rMap.put(ars[0], ars[1]) + }else{ + rMap.put(ars[0], null) + } + } + } + if(rMap){ + count++ + def params = rs.getObject("parameter_map") + println params + def r = [rMap.city, rMap.region, rMap.name] + if(!rets.contains(r)){ + rets << r + } + } + } + }else{ + println msg + } + } + } + println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size() + return rets; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy new file mode 100644 index 000000000..8d353b9b3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy @@ -0,0 +1,131 @@ +package com.fangjia.script.gj.es + +import com.fangjia.core.RegionData +import com.fangjia.core.util.HttpRequest +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.JacksonUtil +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.zip.GZIPInputStream + +class EsHouseMatchDistrict { + + def targetUrl = "http://192.168.0.206:9200/seawater-2021/_doc" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + + static main(args) { + def task = new EsHouseMatchDistrict() + task.houseRun() + } + + def houseRun(){ + def citys = ["杭州"]//CommonUtils.findAllCity(); + for(def city : citys){ + def endDate = DateUtils.parseFromTightByPattern("2021-06-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()) + def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()-1) + def houses = findHouseCounts(city, startDate, endDate); + match(city, houses); + } + } + + + def match(city, houses){ + final CountDownLatch latch = new CountDownLatch(houses.size()) + final AtomicInteger count = new AtomicInteger(0) + final AtomicInteger successCount = new AtomicInteger(0) + for(def i=0;i < houses.size();i++){ + def t = houses[i] + fixedThreadPool.execute( + new Runnable() { + void run() { + count.addAndGet(1) + try{ + if(!t.match_seaweed_id){ + def rMap = SeaweedDataMatchProcess.doMatch(t.city, t.region, t.district_name, null); + if(rMap){ + successCount.addAndGet(1) + updateHouseMatchDistrictId(t.id, rMap.id) + println successCount.get()+" "+t.id+" "+ rMap + } + + } + }catch(Exception e){ + e.printStackTrace() + }finally { + latch.countDown() + } + + } + } + ); + } + latch.await() + if(houses){ + def pre = DataHandleUtils.getDoubleRound(1, successCount.get()*100.0/houses.size()); + println city +" "+houses.size()+" "+new Date().toString()+" "+pre+"%" + } + + + } + + def findHouseCounts(def city, def startDate, def endDate){ + QueryBuilder date_range = QueryBuilders.rangeQuery("publish_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + .must(QueryBuilders.termQuery("cat", "sell")) + .must(date_range).mustNot(QueryBuilders.existsQuery("match_seaweed_id")) +// BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); +// for(def s : ["sell", "rent"]){ +// inBuilder.should(QueryBuilders.matchQuery("cat", s)); +// } +// queryBuilder.must(inBuilder) + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + println houses.size() + def rets = [] + for(def r : houses){ + if(!r.match_seaweed_id){ + rets << r + } + } + return rets + } + + + def updateHouseMatchDistrictId(def id, def match_id){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.match_seaweed_id='${match_id}'", + "lang": "painless" + }, + "query": { + "term": { + "_id": "${id}" + } + } + } + """ + HttpRequest request = new HttpRequest() + String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + println s + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy new file mode 100644 index 000000000..3a572c7ad --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -0,0 +1,817 @@ +package com.fangjia.script.gj.level + +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.BaseBatch +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.JacksonUtil +import com.mongodb.DBObject +import groovy.sql.Sql +import net.sf.json.JSONObject +import org.apache.commons.httpclient.util.DateUtil +import org.apache.commons.lang.StringUtils +import org.apache.commons.math.stat.descriptive.moment.StandardDeviation +import org.apache.commons.math3.stat.descriptive.moment.Mean +import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.zip.GZIPInputStream + +/** + * 小区潜力指数 + */ +class DistrictIndexBatch extends BaseBatch{ + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/" + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static def month = DateUtils.formatDateByPattern(new Date(),"yyyy-MM") + + static main(args) { + runBatch(new DistrictIndexBatch(), args); + } + + @Override + public void process(Object args) { + run(); + } + + def run(){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"] + //def citys = CommonDatautils.findAllCity(); + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2","household_count"]) + def code = cityCodeDatas.get(city) + def city_avg_total_price = null + if(city!="上海"){ + city_avg_total_price = getAvgTotalPrice(city, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) + if(city_avg_total_price){ + city_avg_total_price = city_avg_total_price/10000.0 + city_avg_total_price = DataHandleUtils.getDoubleRound(6, city_avg_total_price) + } + }else{ + city_avg_total_price = 531.2615913 + } + if(city_avg_total_price){ + calcute(city, dists, code, city_avg_total_price) + calcuteIndex(city, dists, code, city_avg_total_price) + + } + println city +" "+city_avg_total_price + } + } + + def calcute(city, dists, citycode, city_avg_total_price){ + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def costDatas = getCostByCity(city) + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def datas = new CopyOnWriteArrayList() + datas << ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] + for(def i=0;i list = DbUtils.findAroundDistrict(d) + def avg_charge = null + def rank = null + if (list) { + list = sortByDistance(d, list); + avg_charge = gtAvgEstateCharge(list) + rank = getPriceInRegionRank(d, list, priceMap) + } + def total_price = getHousePrice(d, dists, priceMap, citycode); + def avg_cost = getAvgCost(d, costDatas) + + def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) + def map = [:] + map.id = d._id.toString() + map.city = d.city + map.region = d.region + map.name = d.name + + map.avg_estate_charge = avg_charge + map.price_rank = rank + map.avg_cost = avg_cost + map.total_price = total_price + //map.comp_value = potentiality + //map.avg_total_price = total_price_index + map.month = month + map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") + + //datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] + + map.sell_avg_price = avg_price + def newUrl = targetUrl +"_doc/"+map.id + httpSaveData(newUrl, map) + println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price+"" + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown(); + } + } + }) + } + latch.await(); + + +// def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] +// def fileName = "/home/gj/potentiality_${city}.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "D:\\数据\\评级\\${city}综合潜力.dat" +// } +// FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false); +// // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") + } + + def calcuteIndex(city, dists, citycode, city_avg_total_price){ + def detailMap = findAllDistrictDatas(city) + def avgMap = calcuteAvgValues(detailMap); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def avg_total_price = avgMap.avg_price + def avg_charge = avgMap.avg_charge + def avg_cost = avgMap.avg_cost + def avg_rank = avgMap.avg_rank + + def sd_avg_price = avgMap.sd_avg_price + def sd_avg_charge = avgMap.sd_avg_charge + def sd_avg_cost = avgMap.sd_avg_cost + def sd_avg_rank = avgMap.sd_avg_rank + + for(def i=0;i + if(v.total_price){ + prices << v.total_price + } + if(v.price_rank){ + ranks << v.price_rank + } + if(v.avg_cost){ + costs << v.avg_cost + } + if(v.avg_estate_charge){ + charges << v.avg_estate_charge + } + } + + def avg_price = null + def dx_avg_price = null + def sd_avg_price = null + if(prices){ + def v = prices.sum()/prices.size() + avg_price = DataHandleUtils.getDoubleRound(6, v) + + double pow_sum = 0 + prices.each {p-> + def sod = Math.abs(p - avg_price) + pow_sum = Math.pow(sod, 2) + pow_sum + } + dx_avg_price = DataHandleUtils.getDoubleRound(6,pow_sum/prices.size()) + sd_avg_price = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_price)) + } + def avg_rank = null + def dx_avg_rank = null + def sd_avg_rank = null + if(ranks){ + def v = ranks.sum()/ranks.size() + avg_rank = DataHandleUtils.getDoubleRound(6, v) + + def pow_sum = 0; + ranks.each {p-> + def sod = Math.abs(p - avg_rank) + pow_sum = Math.pow(sod, 2) + pow_sum + } + dx_avg_rank = DataHandleUtils.getDoubleRound(6,pow_sum/ranks.size()) + sd_avg_rank = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_rank)) + } + + def avg_cost = null + def dx_avg_cost = null + def sd_avg_cost = null + if(costs){ + def v = costs.sum()/costs.size() + avg_cost = DataHandleUtils.getDoubleRound(6, v) + + double pow_sum = 0; + costs.each {p-> + def sod = Math.abs(p - avg_cost) + pow_sum = Math.pow(sod, 2) + pow_sum + } + dx_avg_cost = DataHandleUtils.getDoubleRound(6,pow_sum/costs.size()) + sd_avg_cost = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_cost)) + } + + + def avg_charge = null + def dx_avg_charge = null + def sd_avg_charge = null + + if(charges){ + def v = charges.sum()/charges.size() + avg_charge = DataHandleUtils.getDoubleRound(6, v) + + def pow_sum = 0; + charges.each {p-> + def sod = Math.abs(p - avg_charge) + pow_sum = Math.pow(sod, 2) + pow_sum + } + dx_avg_charge = DataHandleUtils.getDoubleRound(6,pow_sum/charges.size()) + sd_avg_charge = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_charge)) + } + + def avgMap = [:] + avgMap.put("avg_rank", avg_rank) + avgMap.put("avg_price", avg_price) + avgMap.put("avg_cost", avg_cost) + avgMap.put("avg_charge", avg_charge) + + avgMap.put("dx_avg_rank", dx_avg_rank) + avgMap.put("dx_avg_price", dx_avg_price) + avgMap.put("dx_avg_cost", dx_avg_cost) + avgMap.put("dx_avg_charge", dx_avg_charge) + + avgMap.put("sd_avg_rank", sd_avg_rank) + avgMap.put("sd_avg_price", sd_avg_price) + avgMap.put("sd_avg_cost", sd_avg_cost) + avgMap.put("sd_avg_charge", sd_avg_charge) + return avgMap; + } + + //方差s^2=[(x1-x)^2 +...(xn-x)^2]/n 或者s^2=[(x1-x)^2 +...(xn-x)^2]/(n-1) + public double Variance(def x) { + int m=x.size(); + double sum=0; + for(int i=0;i="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def avg_total_price = null + while (rs.next()){ + avg_total_price = rs.getDouble("avg_total_price") + } + if(avg_total_price){ + avg_total_price = avg_total_price.toInteger() + } + return avg_total_price; + } + + def getHousePrice(d, dists, priceMap, citycode){ + //def areas = findHouse(d, 20200101, 20220101) + +// def areas = [] +// if(citycode){ +// areas = findHouseByOracle(d._id.toString(), citycode); +// }else{ +// areas = findHouse(d, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) +// } + def avg_area = findHouse(d, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) + if(!avg_area){ + avg_area = findHousByRegion(d.city, d.region, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) +// if(d.finish_building_date){ +// def year = CommonUtils.getBuildYear(d.finish_building_date) +// if(year){ +// def startYear = (year.toString().substring(0,3)+"0").toInteger() +// def endYear = startYear + 10 +// +// def selectDists = [] +// for(def ad : dists){ +// if(d.region == ad.region){ +// def adYear = CommonUtils.getBuildYear(ad.finish_building_date) +// if(adYear){ +// adYear = adYear.toInteger() +// if(adYear>=startYear && adYear<=endYear){ +// selectDists << ad +// } +// } +// } +// } +// +// if(selectDists){ +// println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size() +// def allAreas = [] +// def ids = [] +// if(citycode){ +// for(def sd : selectDists){ +// ids << sd._id.toString() +// } +// if(ids.size()>=901){ +// ids = ids[0..900] +// } +// +// def idstr = ids.join("','") +// allAreas = findHouseByOracle2(idstr,citycode); +// }else{ +// for(def sd : selectDists){ +// def areaDatas = findHouse(sd, 20190101, 20220101) +// allAreas.addAll(areaDatas) +// } +// } +// if(allAreas){ +// avg_area = allAreas.sum()/allAreas.size(); +// } +// } +// } +// } + } + + if(avg_area){ + def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) + if(avg_price){ + def total_price = (avg_price * avg_area)/10000.0 + def total = DataHandleUtils.getDoubleRound(2, total_price) + return total + } + } + return null + } + + + def gtAvgEstateCharge(list){ + def rets = [] + for(def i=0;i=3){ + break; + } + } + } + if(rets){ + def avg_charge = rets.sum()/rets.size(); + avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge) + return avg_charge; + } + return null + } + + def getPriceInRegionRank(d, nbLists, priceMap){ + if(nbLists){ + if(nbLists.size()==1){ + return 0.5; + }else{ + def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) + def count = 0 + if(avg_price){ + for(def nb : nbLists){ + def price = priceMap.get(nb.city+"##"+nb.region+"##"+nb.name) + if(price > avg_price){ + count ++ + } + } + } + def pre = (nbLists.size() - count)/nbLists.size() + pre = DataHandleUtils.getDoubleRound(2, pre) + return pre; + } + } + return null; + } + + def getCostByCity(city){ + def datas = queryPoi(city) + return datas + } + + def sortByDistance(d, list){ + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + } + //排序 + Collections.sort(list, new Comparator() { + @Override + int compare(Object o1, Object o2) { + if(o1.distance > o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + def getAvgCost(d, costDatas){ + if(costDatas){ + def costs = [] + for(def c : costDatas){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, c.lat, c.lng) + if(distance <= 1000){ + costs << c.cost + } + } + if(costs){ + def avg_cost = costs.sum()/costs.size() + avg_cost = DataHandleUtils.getDoubleRound(2, avg_cost) + return avg_cost + } + + } + return null + } + + + def queryPoi(def city){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode,business.keytag as keytag,business.cost as cost FROM amap_poi_detail WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "市'") + sqlBuffer.append(" and business.cost>20 and business.cost<500") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def datas = [] + def tags = ["法国菜","东南亚菜","日本料理","火锅","海鲜","自助餐","西餐","浙菜","茶餐厅","酒吧","烤肉","潮州菜","云贵菜","韩国料理","中餐","特色火锅","西北菜","串串香","农家乐","烧烤","茶座","上海菜","意大利菜","素菜馆","蛋糕店","东北菜","湘菜","粤菜","咖啡厅","轰趴馆","清真菜","北京菜","川菜","餐饮服务,餐饮相关场所,餐饮相关"] + def index = 0 + while (rs.next()) { + index++ + def tag = rs.getString("keytag") + if(tags.contains(tag)){ + def dMap = [:] + dMap.cost = rs.getInt("cost") + def location = rs.getObject("location") + if(location){ + dMap.lng = location.x + dMap.lat = location.y + } + datas << dMap + }else{ + println tag + } + } + println index + return datas; + } + + + + def findHouseByOracle(def distId, def citycode){ + + def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='${citycode}' AND COMMID='${distId}'"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + allRows << r.BUILDAREA + }) + return allRows + } + + def findHouseByOracle2(def idstr, def citycode){ + + def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='${citycode}' AND COMMID in ('${idstr}')"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + allRows << r.BUILDAREA + }) + return allRows + } + + def findHouse(def dist, def startDate, def endDate){ + def nameQuery = getAllName(dist); + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT avg(area) as avg_area FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+dist.city+"'") + sqlBuffer.append(" and region = '"+dist.region+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + sqlBuffer.append(" and (" + nameQuery + ")"); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def areas = [] + def avg_area = null + while (rs.next()){ +// areas << rs.getDouble("area") + avg_area = rs.getDouble("avg_area") + } + if(avg_area){ + avg_area = DataHandleUtils.getDoubleRound(2, avg_area) + } + return avg_area; + } + + + def findHousByRegion(def city, def region, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT avg(area) as avg_area FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and region = '"+region+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def avg_area = null + while (rs.next()){ + avg_area = rs.getDouble("avg_area") + } + if(avg_area){ + avg_area = DataHandleUtils.getDoubleRound(2, avg_area) + } + return avg_area; + } + + def getAllName(dist){ + Set aliasNames = new LinkedHashSet(); + aliasNames.add(dist.name); + if (dist != null && dist.alias != null) { + aliasNames.addAll(dist.alias); + } + + List namePars = new ArrayList(); + if(aliasNames!=null && aliasNames.size()>0) { + for(String name : aliasNames){ + if(name && name.trim().length() > 1){ + namePars.add("district_name ='" + name + "'"); + } + } + } + + if(namePars.size() > 0){ + return namePars.join(" OR ") + } + return null; + } + + + +// def updateHouseMatchDistrictId(def id, def match_id){ +// def header = [:] +// header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() +// String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" +// def body = """ +// { +// "script": { +// "source": "ctx._source.match_seaweed_id='${match_id}'", +// "lang": "painless" +// }, +// "query": { +// "term": { +// "_id": "${id}" +// } +// } +// } +// """ +// HttpRequest request = new HttpRequest() +// String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") +// println s +// } + + + + def httpSaveData(def newUrl,def params){ + def header = [:] + header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=") + doPost(newUrl,header, params, true, "utf-8") + } + + + def doPost(def targetUrl, def headers, def params, def isJson, def charset){ + StringBuilder buffer = new StringBuilder(); + try{ + URL url = new URL(targetUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setUseCaches(false); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + conn.setRequestProperty("User-Agent", UserAgents.get()); + String param = null; + BufferedWriter mWriter = null; + InputStreamReader inStream = null; + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + conn.setRequestProperty(head, headers.get(head)); + } + } else { + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" ); + } + + if (isJson) { + conn.setRequestProperty("Content-Type", "application/json; charset=" + charset); + param = JacksonUtil.beanToJson(params) + } + + conn.connect(); + + //println param.toString() + //输出流 + if (StringUtils.isNotEmpty(param)) { + mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset)); + mWriter.write(param); + mWriter.flush(); + mWriter.close(); + } + if (conn.getHeaderField("Content-Encoding") != null + && conn.getHeaderField("Content-Encoding").equals("gzip")) { + inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset); + } else { + inStream = new InputStreamReader(conn.getInputStream(), charset); + } + char[] bf = new char[1024]; + int len; + while ((len = inStream.read(bf)) != -1) { + buffer.append(bf, 0, len); + } + } catch (Exception e) { + e.printStackTrace() + } + return buffer.toString(); + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy new file mode 100644 index 000000000..9ef7e6926 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy @@ -0,0 +1,215 @@ +package com.fangjia.script.util + +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.client.config.RequestConfig +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.action.ActionListener +import org.elasticsearch.action.bulk.BackoffPolicy +import org.elasticsearch.action.bulk.BulkProcessor +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.ClearScrollRequest +import org.elasticsearch.action.search.SearchRequest +import org.elasticsearch.action.search.SearchResponse +import org.elasticsearch.action.search.SearchScrollRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.function.BiConsumer + +/** + * 功能说明:Elasticsearch工具类 + */ +class DsElasticUtils { + + + static getEsDataSource(String host){ + EsDataSource dataSource = new EsDataSource (); + dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"); + Properties connectionProperties = new Properties(); + connectionProperties.setProperty("user","developer"); + connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); + dataSource.setProperties(connectionProperties); + return dataSource; + } + + + static RestHighLevelClient getClient(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + + + /** + * 创建一个ES批处理器 + * @param host + * @return + */ + static createBulkProcessor(String host, RestHighLevelClient client = null) { + + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + @Override + void beforeBulk(long executionId, BulkRequest request) { + } + + @Override + void afterBulk(long executionId, BulkRequest request, + BulkResponse response) { + } + + @Override + void afterBulk(long executionId, BulkRequest request, + Throwable failure) { + } + }; + if(!client){ + client = getClient(host); + } + return BulkProcessor.builder(new BiConsumer>() { + @Override + void accept(BulkRequest bulkRequest, ActionListener bulkResponseActionListener) { + client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener) + } + }, listener).setBulkActions(10000) + .setConcurrentRequests(20) + .setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)) + .setFlushInterval(TimeValue.timeValueSeconds(10L)).build(); + } + + /** + * 关闭一个ES批处理器 + * @param + * @return + */ + static closeBulkProcessor(BulkProcessor bulkProcessor) { + bulkProcessor.awaitClose(10, TimeUnit.DAYS) + bulkProcessor.flush() + bulkProcessor.close() + } + + + static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){ + + RestHighLevelClient client = getClient(host) + BulkProcessor bulkProcessor = createBulkProcessor(host, client) + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(qb); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + def nThread = Runtime.getRuntime().availableProcessors() * 5 + ExecutorService executorService = Executors.newFixedThreadPool(nThread) + while (searchHits != null && searchHits.length > 0) { + for (int k=0; k < searchHits.size(); k++) { + final SearchHit hit = searchHits[k]; + executorService.submit(new Runnable() { + @Override + void run() { + try{ + if(closure.getMaximumNumberOfParameters() == 1){ + def update = closure.call(hit.getSourceAsMap()) + if(update){ + bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) + } + }else if(closure.getMaximumNumberOfParameters() == 2){ + def update = closure.call(hit.getId(), hit.getSourceAsMap()) + if(update){ + bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) + } + } + }catch (Exception e){ + e.printStackTrace() + } + } + }) + } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + executorService.shutdown() + executorService.awaitTermination(1, TimeUnit.DAYS) + bulkProcessor.close() + bulkProcessor.awaitClose(1, TimeUnit.DAYS) + client.close() + } + + + static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){ + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) + RestHighLevelClient client = getClient(host) + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(qb); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + def results = [] + while (searchHits != null && searchHits.length > 0) { + for (int k=0; k < searchHits.size(); k++) { + SearchHit searchHit = searchHits[k] + searchHit.getSourceAsMap().put("id", searchHit.getId()); + results << searchHit.getSourceAsMap() + } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + client.close() + return results; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy index 0ad5dbfcf..362bf2afb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy @@ -1,11 +1,20 @@ package com.fangjia.script.gj.location +import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.codehaus.jackson.sym.NameN import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet - +import com.fangjia.script.gj.district.FangjiaClientUtils class DistrictLocationUpdate { public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" @@ -19,20 +28,214 @@ class DistrictLocationUpdate { static main(args) { def task = new DistrictLocationUpdate() - task.run() +// task.run() + task.mapMatch() +// task.updateLocationToDB() } def run(){ - def citys = CommonDatautils.findAllCity() + def citys = ["福州"]//CommonDatautils.findAllCity() for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"]) - - - updateLocation(city, dists); + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"]) + def rets = [] + rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"] + for(def d : dists){ + def lng = null + def lat = null + if(d.lng){ + lng = d.lng/10000000000.0 + lat = d.lat/10000000000.0 + } + rets << [d.city, d.region, d.name, d.address, lng,lat] + } + ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据") } } + def mapMatch(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d-> + if(!d['H'] && d['C']=="华侨城龙居御景二期"){ + index++ +// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德") +// println d['A']+" "+d['B']+" "+d['C'] +// if(gdData){ +// d['G'] = gdData.district +// d['H'] = gdData.name +// d['I'] = gdData.address +// d['J'] = gdData.type +// d['K'] = gdData.lng +// d['L'] = gdData.lat +// d['M'] = gdData.score +// if(d['E'] && d['K']){ +// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble()) +// d['N'] = distance +// } +// } + + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); + if (!map || !map.lat) { + rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); + map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); + } + if(map){ + if(map.adname){ + d['G'] = map.adname + }else if(map.region){ + d['G'] = map.region + } + d['H'] = map.name + d['I'] = map.address + if(map.type){ + d['J'] = map.type + }else if(map.estateType){ + d['J'] = map.estateType + } + d['K'] = map.lng + d['L'] = map.lat + d['M'] = map.score + if(d['E'] && d['K']){ + def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble()) + d['N'] = distance + } + } + } + },0) + } + + def updateLocationToDB(){ + ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d-> + if(d['N'] && d['N']>1000){ + if(d['M']>=0.4 && d['K']){ + def query = new BasicDBObject() + query.put("city", d['A']) + query.put("region", d['B']) + query.put("name", d['C']) + + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + + Long lng0 = d['K'].toString().toDouble()*10000000000L + Long lat0 = d['L'].toString().toDouble()*10000000000L + udata.put("lat", lat0) + udata.put("lng", lng0 ) + + def gmap = GeocodeUtils.getGaodeToBaidu(d['K'].toString().toDouble(), d['L'].toString().toDouble()) + Long lng2 = gmap.get("lng")*10000000000L + Long lat2 = gmap.get("lat")*10000000000L + udata.put("lat2", lat2) + udata.put("lng2", lng2) + DbUtils.seaweed.update(query, new BasicDBObject('$set', udata)) + } + } + },0) + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } + + def findStandardObject(def city, def region, def name, def rets) { + def map; + if (rets) { + if (rets.size() >= 1) { + for (def mapObj : rets) { + if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){ + continue; + } +// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){ + map = mapObj; + break; + }else if(mapObj.address && mapObj.address.contains(name)){ + map = mapObj; + break; + }else if(mapObj.name && name){ + double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name); + if(score1){ + map = mapObj; + break; + } + } +// } + } + } + } + return map; + } + + def queryPoiMap(def city){ final StringBuffer sqlBuffer = new StringBuffer() sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy new file mode 100644 index 000000000..47c9fe748 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictMapMatch.groovy @@ -0,0 +1,123 @@ +package com.fangjia.script.gj.location + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils + +class DistrictMapMatch { + + static main(args) { + def task = new DistrictMapMatch() + task.mapMatch() + } + + + def mapMatch(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\重庆农商行\\小区\\待审核小区(地图匹配).xlsx",{ d-> + if(!d['H']){ + index++ + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// if (!map || !map.lat) { +// rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); +// map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); +// } + if(map){ + d['F'] = map.id + if(map.adname){ + d['G'] = map.adname + }else if(map.region){ + d['G'] = map.region + } + d['H'] = map.name + d['I'] = map.address + if(map.type){ + d['J'] = map.type + }else if(map.estateType){ + d['J'] = map.estateType + } + d['K'] = map.lng + d['L'] = map.lat + d['M'] = map.score + d['N'] = map.source + } + } + },0) + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index ce1d3ee1a..3c2061a52 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -40,9 +40,9 @@ class LocationTest { // System.out.println(str); def task = new LocationTest() -// task.run() + task.run() // task.run2() - task.run3() +// task.run3() // task.run4() // task.run5() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy new file mode 100644 index 000000000..f42431df6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/meituan/MatchTest.groovy @@ -0,0 +1,402 @@ +package com.fangjia.script.gj.meituan + +import com.fangjia.core.RegionData +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.gj.paimai.DataUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.bson.types.ObjectId + +class MatchTest { + static main(args) { + def task = new MatchTest() +// task.run() +// task.addNum() +// task.dongguang() +// task.meituanDel() + + task.addLevel() + } + + + + def addLevel() { + def citys = ["昆明","南宁","郑州","长沙","贵阳","乌鲁木齐","中山","湛江","南昌","海口"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + } + + + def path = "D:\\数据\\美团\\10城小区明细-结果.xlsx"//美团金融小区测试list-所有数据-评级.xlsx + ExcelUtils.multiThreadingModify(path, { row -> + + def name = row['D'] + def address = row['E'] + def city = RegionData.formatCity(row['A']) + + if(name && !row['O']){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } +// if(rMap.level){ +// row['Q'] = rMap.level +// } + } + } + } + + }) + } + + + def meituanDel(){ + def path = "D:\\数据\\美团\\美团需入库小区合并后.xlsx" + def index = 0 + ExcelUtils.modify(path, { row -> + def flag = row['O'] + if(flag){ + def city = row['F'] + def region = row['G'] + def name = nameFileter(row['H']) + def obj = null + if(city == "东莞"){ + obj = CommonDatautils.findDistrict(city, null, name, "district") + }else{ + obj = CommonDatautils.findDistrict(city, region, name, "district") + } + if(obj){ + index++ + println index +" "+ city +' '+ region+" "+name + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("status", 1) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(obj._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + } + },0); + } + + def dongguang(){ + def path = "D:\\数据\\美团\\美团东莞.xlsx"//美团金融小区测试list-匹配.xlsx + ExcelUtils.modify(path, { row -> + def region = null + if(row['B'] && row['B']=="NULL" ){ + region = null + } + def name = nameFileter(row['H']) + def address = row['E'] + if (name && name.contains("湖畔新家")) { + + def obj = CommonDatautils.findDistrict("东莞", "东莞", name, "district") + if(obj){ + def json = GaoDeApiUtils.geo(row['F'], address) + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + def twinshop = getStreetInfo(m.location) + if(twinshop){ + def newRegion = RegionData.formatRegion("东莞",twinshop) + if(newRegion){ + def dbObj = CommonDatautils.findDistrict("东莞", newRegion, name, "district") + if(dbObj){ + println "delete" + DbUtils.seaweed.remove(new BasicDBObject("_id",obj._id)) + }else{ + println "update" + def udata = new BasicDBObject() + udata.put("region", newRegion) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata)) + } + } + } + // println address +" "+twinshop + } + } + } + } + },0) + + } + + def getStreetInfo(def glocation){ + def township = null + if(glocation){ + def jsonObject = GaoDeApiUtils.geocode(glocation) + if(jsonObject && jsonObject.regeocode) { + def addressComponent = jsonObject.regeocode.addressComponent + if(addressComponent){ + township = addressComponent.township + } + } + } + return township; + } + + def addNum(){ + def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据.xlsx"//美团金融小区测试list-匹配.xlsx + def index = 0 + ExcelUtils.modify(path, { row -> + index++ + row['A'] = index + },0) + } + + def run() { + def citys = ["潮州","金华","惠州","东莞","苏州","桂林","廊坊","临沧","通辽","雅安"]// + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def cityLevelMap = [:] + def radioMap = [:] + for(def city : citys){ + +// def lastMonthDay = DateUtil.truncateDay(DateUtil.getLastDayOfMonth(new Date(), -1)); +// def avg_radio = CommonDatautils.findCityAvgRentRadio(city, lastMonthDay) +// radioMap.put(city, avg_radio) + + + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def datas = addPriceData(dists, priceMap); + datas = calcuteLevel(datas); + cityLevelMap.put(city, ["price":priceMap, "level":datas]) + println city +" 楼盘评级同步完成" + + } + + + def path = "D:\\数据\\美团\\美团金融小区测试list-所有数据-评级.xlsx"//美团金融小区测试list-所有数据-评级.xlsx + ExcelUtils.multiThreadingModify(path, { row -> + def region = null + if(row['C'] && row['C']=="NULL" ){ + region = null + } + def name = row['D'] + def address = row['F'] + def city = RegionData.formatCity(row['B']) + + if(name && !row['N']){ + if(city){ + def map = cityLevelMap.get(city) + def rMap = null + if(map){ + rMap = getArroundDistrictPriceByAddress(city, address, map.level) + } + + if(rMap){ + if(rMap.rank){ + row['Q'] = rMap.rank + } + if(rMap.level){ + row['P'] = rMap.level + } + } + } + } + + if (name && row['N']) { +//// def jsonObject = OpenApiClient.addressMatch(row['A'], address, null, true, region, name) +// def jsonObject = OpenApiClient.addressMatch(row['A'], name, null, true, region, null) +// if (jsonObject != null && jsonObject.precise != -1) { +// row['F'] = jsonObject.get("city") +// row['G'] = jsonObject.get("district") +// row['H'] = jsonObject.get("name") +// row['I'] = jsonObject.get("propertyType") +// row['J'] = jsonObject.get("map") +// row['K'] = jsonObject.get("lng") +// row['L'] = jsonObject.get("lat") +// } + + if(row['G'] && row['H'] && row['I']){ +// def priceObj = CommonDatautils.findDistrictPrice(row['F'], row['G'], row['H']) +// if(priceObj){ +// row['M'] = priceObj.avg_price +// } +// def sqmetreRentPrice = CommonDatautils.findNewSqmetreRent(row['F'], row['G'], row['H']) +// if(sqmetreRentPrice){ +// row['N'] = sqmetreRentPrice +// } + + if(row['N'] && !row['O']){ + def avg_rent_radio = radioMap.get(city) + row['O'] = (int)(row['N']/avg_rent_radio) + } + } + + } + }) + } + + def getArroundDistrictPriceByAddress(def city, def address, def levels){ + def json = GaoDeApiUtils.geo(city, address) + def location = null + def detail = new BasicDBObject() + def level = null + def rank = null + if(json && json.count>=1){ + def m = json.geocodes[0] + if(m){ + location = m.location + level = m.level + } + } + if(location){ + def arrs = location.split(",") + detail.lng = arrs[0].toString().toDouble()*10000000000L + detail.lat = arrs[1].toString().toDouble()*10000000000L + } + if(detail.lng){ + detail.city = city + + List list = DbUtils.findAroundDistrictByRadius(detail, 3000) + if(list && list.size()>10){ + list = sortByDistance(detail, list); + list = list[0..9] + } + def arroundAvgPrice = getArroundAvgPrice(list) + if(arroundAvgPrice){ + + def mod = Integer.MAX_VALUE; + for(def d : levels){ + def newMod = java.lang.Math.abs( d.avg_price - arroundAvgPrice) + if(newMod o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } + }) + return list; + } + + def addPriceData(dists, priceMap){ + def newLists = [] + for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); + if(price){ + d.avg_price = price; + newLists << d; + } + } + if(newLists){ + newLists.sort {a,b-> + def v1 = a.avg_price + def v2 = b.avg_price + return v1.compareTo(v2) + } + } + return newLists; + } + + def calcuteLevel(def datas){ + def total = datas.size() + def v = (int)(total/10) + def start = 0; + def end = 0; + def map = [:] + for(int i=1;i<=10;i++){ + end = end + v; + if(i == 10){ + map.put(i, [start, total]) + }else{ + map.put(i, [start, end]) + } + start = end; + } + for(def d=0;dm && d<=n){ + dist.level = 10 - o.key + 1 + break + } + } + } + } + return datas; + } + + def getArroundAvgPrice(def list){ + def prices = [] + for(def l : list){ + def priceObj = CommonDatautils.findDistrictPrice(l.city, l.region, l.name) + if(priceObj){ + prices << priceObj.avg_price + } + } + if(prices){ + def avg_price = (prices.sum()/prices.size()).toInteger() + return avg_price; + } + return null + } + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + return name; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy new file mode 100644 index 000000000..60ae682b9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -0,0 +1,405 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.gj.paimai.ChromeDriverPool +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +public class BeiKeCrawlByChrome { + + def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new BeiKeCrawlByChrome(); + task.run() +// task.run1() + task.addDealRegion(); + } + + + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def citys = getAllCityUrls(driver) + citys = citys.reverse() + def start = 0 +// for(def i=0;i=endDate.getTime()){ + detailUrls << durl + }else{ + stop = true; + break; + } + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + + def getPageDeail(driver, city, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "/chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") + if(dealDate && dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + parse(city, e) + }else{ + stop = true; + break; + } + } + if(page>=100){ + stop = true; + break; + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + def parse(city,e){ + try { + def title = e.select(".title a").text() + if (title) { + def detail = new BasicDBObject() + def arrs = title.split(" ") + detail.city = city + detail.district_name = arrs[0] + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + if(detail.region){ + detail.region = RegionData.formatRegion(city, detail.region) + } + + def roomStr = arrs[1] + if (roomStr) { + if (roomStr.indexOf("室") > 0) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + } + if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + } + } + + def areaStr = arrs[2].replace("平米", "") + if (areaStr && areaStr.isNumber()) { + detail.area = areaStr.toDouble() + } + + def price = e.select(".totalPrice .number").text() + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "贝壳找房" + def detailUrl = e.select(".title a").attr("href") + def dbo = findTrade(detailUrl) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = detailUrl + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } + } + }catch(Exception ex){ + ex.printStackTrace() + println e.select(".title a").attr("href") + } + } + + + def findRegionByThirdPary(city, district_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("name", district_name) + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1) + def dbo = null; + for(def d : dbcursor){ + dbo = d + } + if(dbo){ + return dbo.region + } + return null; + } + def getContent(driver, detailUrl){ + driver.get(detailUrl) + def str = driver.getPageSource() + if(str && str.contains("登录")){ + getLogin(driver) + return getContent(driver, detailUrl) + } + return str; + } + def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){ + if(detailUrls){ + for(def i=0;i=4){ + detail.city = eles[0].text().replace("房产网","") + detail.region = eles[2].text().replace("二手房成交","") + detail.district_name = eles[eles.size()-1].text().replace("二手房成交","") + } + if(detail.district_name) { + def price = JQueryUtils.selectText(".dealTotalPrice i", doc) + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + def dbo = findTrade(DEAL_PRICE_COLL, detail) + if (!dbo) { + def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") + if (roomStr) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger() + } + detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") + detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") + def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "") + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = detailUrl + detail.c_date = new Date() + detail.m_date = new Date() + + DEAL_PRICE_COLL.save(detail) + } + } + return detail; + } + + + def findTrade(def detailUrl){ + def q = new BasicDBObject() + q.put("city", detail.city) +// q.put("region", detail.region) +// q.put("district_name", detail.district_name) +// q.put("area", detail.area) +// q.put("total_price", detail.total_price) + q.put("url", detailUrl) + q.put("trade_date", detail.trade_date) + q.put("source", detail.source) + def dbo = DEAL_PRICE_COLL.findOne(q) + return dbo + } + + def getAllCityUrls(driver){ + driver.get("https://www.ke.com/city/") + Thread.sleep(8000) + def cityDoc = Jsoup.parse(driver.getPageSource()); + def cityEles = JQueryUtils.select(".city_list_ul .city_province a", cityDoc) + def list = [] + if(cityEles){ + for(def e : cityEles){ + def href = e.attr("href") +// if(e.text().contains("福州")){ + def cityMap = [:] + cityMap.city = e.text() + cityMap.url = "https:"+href + list << cityMap + //cityMap.put(e.text(), "https:"+href) +// } + } + } + return list + } + + def getLogin(driver){ + driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); + WebDriverWait wait = new WebDriverWait(driver,20) + Thread.sleep(10000) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + + + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + + + elemUsername.sendKeys("15021384995"); + + elemPassword.sendKeys("12qwaszx"); + + btn.submit(); // 点击行为; + } +} + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy index e2c0d9152..fb358e503 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Caa123Crawl.groovy @@ -15,6 +15,7 @@ import org.openqa.selenium.chrome.ChromeDriver import org.openqa.selenium.support.ui.ExpectedConditions import org.openqa.selenium.support.ui.WebDriverWait +import javax.swing.JApplet import java.text.DateFormat import java.util.function.Consumer @@ -41,45 +42,138 @@ System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Loca System.setProperty("http.nonProxyHosts", "localhost"); -def doc = getDocument(); -def detailUrls = getDetailLinks(doc); -def rets = batchParse(detailUrls); -saveToDB(rets); +WebDriver driver = new ChromeDriver() +def proviceMap = getDocument(driver); +def citys = getCity(driver,proviceMap) +def detailUrls = getDetailLinks(driver, citys); -def getDetailLinks(doc){ - def urls = [] + +def getDetailLinks(driver, cityList){ + + def zxcitys = ["天津市","北京市","上海市","重庆市"] + for(def cityMap : cityList){ + def citys = cityMap.citys + + citys.each{city,id-> + def urls = [] + if(!zxcitys.contains(cityMap.provice)){ + def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&city=${id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=" + getDocument(url, driver) + }else{ + def url = "https://paimai.caa123.org.cn/pages/lots/list.html?&province=${cityMap.provice_id}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=&canLoan=&isRestricted=&insuranceSupport=" + getDocument(url, driver) + } + + def doc = Jsoup.parse(driver.getPageSource()); + def count = JQueryUtils.selectText(".count", doc) + def totalPage = getTotalPage(count, 12) + if(totalPage){ + for(def i=1;i<=totalPage;i++){ + doc = Jsoup.parse(driver.getPageSource()); + def eles = JQueryUtils.select(".lots-list .lot-li-a", doc) + if(eles){ + for(def el :eles){ + def href = el.attr("href") + def map = [:] + map.province = cityMap.provice + map.city = city + map.url = "https://paimai.caa123.org.cn"+href + urls << map + } + } + try{ + driver.findElement(By.className("next")).click() + }catch(Exception e){ + Thread.sleep(8000) + driver.findElement(By.className("next")).click() + } + } + + def rets = batchParse(urls, driver); + saveToDB(rets); + println city +" "+urls.size() + } + } + println cityMap.provice +" 完成" + + } + + //return urls; +} + +def getTotalPage(def total, def pageSize){ + if(total){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + return 0; +} + +def getCity(driver, proviceMap){ + def rows = [] + def zxcitys = ["天津市","北京市","上海市","重庆市"] + proviceMap.each {k,v-> + if(!zxcitys.contains(k)){ + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=${v}&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); + def doc = Jsoup.parse(driver.getPageSource()); + def cityEles = JQueryUtils.select("li:contains("+k+") .value-limited li a", doc) + def cityMap = [:] + for(def c : cityEles){ + def cityid = c.attr("data-cityid") + def city = c.text() + if(!["石家庄市","太原市","齐齐哈尔市","哈尔滨市","鸡西市","长治市","朔州市","廊坊市","衡水市","唐山市","秦皇岛市","邯郸市","邢台市","保定市","沧州市","张家口市","承德市"].contains(city)){ + cityMap.put(city, cityid) + } + + } + def map = [:] + map.provice = k + map.provice_id = v + map.citys = cityMap + rows << map + }else{ + def map = [:] + map.provice = k + map.provice_id = v + def cityMap = [:] + cityMap.put(k, v) + map.citys = cityMap + rows << map + } + } + return rows +} + +def getDocument(driver){ + + driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&secondaryType=1&canLoan=&isRestricted=&insuranceSupport="); + WebDriverWait wait = new WebDriverWait(driver,20) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list")))); + def doc = Jsoup.parse(driver.getPageSource()); + def provinceMap = [:] if(doc){ - def eles = JQueryUtils.select(".lots-list .lot-li-a", doc) - if(eles){ - for(def el :eles){ - def href = el.attr("href") - urls << "https://paimai.caa123.org.cn"+href + def proviceEles = JQueryUtils.select(".auction-locus .limited-value-list", doc) + for(def ele : proviceEles){ + def provice = ele.text() + def value = ele.attr("data-proviceid") + if(value && provice!="北京市" && provice!="天津市" && provice!="吉林省" && provice!="辽宁省" && provice!="内蒙古自治区" && provice!="山西省" && provice!="河北省" && provice!="江苏省" && provice!="黑龙江省"){ + provinceMap.put(provice, value) } } } - return urls; + return provinceMap; } -def getDocument(){ - WebDriver driver = new ChromeDriver() - driver.get("https://paimai.caa123.org.cn/pages/lots/list.html?&province=9&status=&attribute=&term=&startTimeStamp=&endTimeStamp=&standardType=6&canLoan=&isRestricted="); - WebDriverWait wait = new WebDriverWait(driver,20) - wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("lots-list")))); +def batchParse(def urls, driver){ - def doc = Jsoup.parse(driver.getPageSource()); - - - return doc; -} - -def batchParse(def urls){ - WebDriver driver = new ChromeDriver() def ret = [] - for(def url :urls){ - driver.get(url); - def doc = Jsoup.parse(driver.getPageSource()); - def res = parse(url, doc); + for(def map :urls){ + def doc = getDocument(map.url, driver) + def res = parse(map.province, map.city, map.url, doc); if(res){ ret << res } @@ -87,111 +181,167 @@ def batchParse(def urls){ return ret; } -def parse(url,doc){ +def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc +} + +def parse(province, city, url,doc){ if(doc){ def result = [:] -// def address = JQueryUtils.selectText(["#pro_bid_name"], doc) -// result.address = address - if(result.address){ - def rMap = OpenApiUtils.matchDistrict(result.address) - if(rMap) { - result.matchCity = rMap.matchCity - result.matchRegion = rMap.matchRegion - result.matchName= rMap.matchName - result.matchAddress = rMap.matchAddress + result.province = province + result.city = city + def address = JQueryUtils.selectText(["#pro_bid_name"], doc) + result.address = address + if(result.address && !result.address.contains("测试")) { + def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num } - } - def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc) - if(finishDateStr) { - result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss"); - } + result.status = JQueryUtils.selectText([".bd_state"], doc) + result.companyName = JQueryUtils.selectText("p:contains(拍卖企业) .companyName", doc) + result.mobile = JQueryUtils.selectText("p:contains(联系方式) .companyTel", doc) - def curPriceStr = JQueryUtils.selectText([".colorred"], doc) - if(curPriceStr) { - curPriceStr = curPriceStr.replace(",","") - result.tradePrice = curPriceStr.toInteger() - } + def finishDateStr = JQueryUtils.selectText(["#sf-countdown .time"], doc) + if (finishDateStr) { + try{ + result.finishDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm"); + }catch(Exception e){ + + } - def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc) - if(evaluationPriceStr) { - evaluationPriceStr = evaluationPriceStr.replace(",", "") - if(evaluationPriceStr.isInteger()) { - result.evaluationPrice = evaluationPriceStr.toInteger() } - } - def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc) - if(depositPriceStr) { - depositPriceStr = depositPriceStr.replace(",", "") - if(depositPriceStr.isInteger()) { - result.depositPrice = depositPriceStr.toInteger() + def curPriceStr = JQueryUtils.selectText([".colorred"], doc) + if (curPriceStr) { + curPriceStr = curPriceStr.replace(",", "") + result.tradePrice = curPriceStr.toDouble().toInteger() } - } - def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc) - if(addPriceStr) { - addPriceStr = addPriceStr.replace(",", "") - if(addPriceStr.isInteger()) { - result.addPrice = addPriceStr.toInteger() - } - } - - def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc) - if(startPriceStr) { - startPriceStr = startPriceStr.replace(",", "").replace("元", "") - if(startPriceStr.isInteger()) { - result.startPrice = startPriceStr.toInteger() - } - } - - result.preemptivePerson= JQueryUtils.selectText(["#prior_span"], doc) - result.retain = JQueryUtils.selectText(["#retain"], doc) - - def imgs =JQueryUtils.select(".sf-pic-slide img", doc) - if(imgs) { - def imgLinks = [] - for(def img : imgs) { - def href = img.attr("src") - if(href.contains("http")) { - imgLinks << href - }else { - imgLinks << url.substring(0, url.indexOf(":")+1)+href + def evaluationPriceStr = JQueryUtils.selectText(["#assessPrice"], doc) + if (evaluationPriceStr) { + evaluationPriceStr = evaluationPriceStr.replace(",", "") + if (evaluationPriceStr.isNumber()) { + result.evaluationPrice = evaluationPriceStr.toDouble().toInteger() } } - result.imgUrls = imgLinks - } - def desc = JQueryUtils.selectText(["#J_desc"], doc) - if(!desc){ - desc = JQueryUtils.selectText(["#pro_bid_name"], doc) - } - if(desc) { - if(desc.contains("别墅")) { - result.address = desc.substring(0, desc.indexOf("别墅")+2) - }else if(desc.contains("号楼")) { - result.address = desc.substring(0, desc.indexOf("号楼")+2) - }else if(desc.contains("房屋")) { - result.address = desc.substring(0, desc.indexOf("房屋")+2) + def depositPriceStr = JQueryUtils.selectText(["#cashDeposit"], doc) + if (depositPriceStr) { + depositPriceStr = depositPriceStr.replace(",", "") + if (depositPriceStr.isNumber()) { + result.depositPrice = depositPriceStr.toDouble().toInteger() + } } - } - def descr = JQueryUtils.selectText(["body"], doc) - if(descr) { - def area = getAreaNumber(descr, 5000); - if(area){ - result.area = area.toDouble() + def addPriceStr = JQueryUtils.selectText(["#rateLadder"], doc) + if (addPriceStr) { + addPriceStr = addPriceStr.replace(",", "") + if (addPriceStr.isNumber()) { + result.addPrice = addPriceStr.toDouble().toInteger() + } } - } - if(result.area && result.tradePrice){ - result.tradeAvgPrice = (int)(result.tradePrice/result.area) - } - result.content = DataUtil.zip(doc.toString()) - result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() - result.url = url - return result; + def startPriceStr = JQueryUtils.selectText(["#startPrice"], doc) + if (startPriceStr) { + startPriceStr = startPriceStr.replace(",", "").replace("元", "") + if (startPriceStr.isNumber()) { + result.startPrice = startPriceStr.toDouble().toInteger() + } + } + + result.preemptivePerson = JQueryUtils.selectText(["#prior_span"], doc) + result.retain = JQueryUtils.selectText(["#retain"], doc) + result.lookCount = JQueryUtils.selectText(".look_count span", doc) + + def startDateStr = JQueryUtils.selectText("span:contains(拍卖会定于)", doc) + if(startDateStr && startDateStr.endsWith("定于") && startDateStr.endsWith("日")){ + startDateStr = startDateStr.toString().substring(startDateStr.indexOf("定于")+2, startDateStr.indexOf("日")+1).trim() + } + + try{ + result.startDate = com.fangjia.script.util.DateUtils.parseString2Date(startDateStr.trim()) + }catch(Exception e){ + + } + + def imgs = JQueryUtils.select(".sf-pic-slide img", doc) + if (imgs) { + def imgLinks = [] + for (def img : imgs) { + def href = img.attr("src") + if (href.contains("http")) { + imgLinks << href + } else { + imgLinks << url.substring(0, url.indexOf(":") + 1) + href + } + } + result.imgUrls = imgLinks + } + + def desc = JQueryUtils.selectText(["#J_desc"], doc) + if (!desc) { + desc = JQueryUtils.selectText(["#pro_bid_name"], doc) + } + if (desc) { + if (desc.contains("别墅")) { + result.address = desc.substring(0, desc.indexOf("别墅") + 2) + } else if (desc.contains("号楼")) { + result.address = desc.substring(0, desc.indexOf("号楼") + 2) + } else if (desc.contains("房屋")) { + result.address = desc.substring(0, desc.indexOf("房屋") + 2) + } + } + + def descr = JQueryUtils.selectText(["body"], doc) + if (descr) { + def area = getAreaNumber(descr, 5000); + if (area) { + result.area = area.toDouble() + } + } + if (result.area && result.tradePrice) { + result.tradeAvgPrice = (int) (result.tradePrice / result.area) + } + + if(result.finishDate){ + result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") + } + if(result.startDate){ + result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01") + } + + result.content = DataUtil.zip(doc.toString()) + result.introduce = JQueryUtils.select([".d-block table:eq(1)"], doc).toString() + result.url = url + return result; + } } return null } @@ -207,11 +357,13 @@ def saveToDB(rets){ ret.each{k,v-> obj.put(k,v) } - + obj.source= "中拍网" if(dbo){ - obj.remove("url") + obj.m_date = new Date() paimaiResultColl.update(q, new BasicDBObject('$set', obj)) }else{ + obj.c_date = new Date() + obj.m_date = new Date() paimaiResultColl.save(obj) } } @@ -229,16 +381,25 @@ def getAreaNumber(input, maxArea) { def area = null; for (def val : list) { + if(val.toString().contains(" ")){ + val = val.toString().substring(val.toString().indexOf(" ")+1, val.toString().size()) + } + if(val.contains("室")){ + val = val.toString().substring(val.toString().indexOf("室")+1, val.toString().size()) + } def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", ""); - if (maxArea) { - if (v.toDouble() < maxArea) { - area = v; + if(v && v.toString().isNumber()){ + if (maxArea) { + if (v.toDouble() < maxArea) { + area = v; + break; + } + } else { + area = v break; } - } else { - area = v - break; } + } if (area) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy new file mode 100644 index 000000000..53c9378bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.gj.paimai + +import org.openqa.selenium.chrome.ChromeDriver + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +public class ChromeDriverPool { + + private static ConcurrentHashMap pool ;//线程池 + private static int coreSize = 2;//核心线程池,一直驻留的资源 + private static int maxSize = 10;//TODO 设置最大数量资源,用于解决瞬时并发,瞬时并发过后要定时去识别空闲的释放掉,保留coreSize的资源 + private static AtomicInteger count = new AtomicInteger(0);//原子自增!! + + static { + //驱动初始化 + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + pool = new ConcurrentHashMap<>(); + } + + private ChromeDriverPool(){ + } + + public static void init(int coreSize){//线程池数量初始化 + ChromeDriverPool.coreSize = coreSize; + } + + public static Worker getPool(){ + if(count.getAndIncrement() < coreSize){//自增!!! + return push(new Worker()); + }else{ + return pop(); + } + } + + + public static Worker push(Worker worker){ + pool.put(count.get(),worker); + return worker; + } + + /** + * 循环获取未使用的资源 + * @return + */ + public static Worker pop(){ + Set> sets = pool.entrySet(); + Iterator> its = sets.iterator(); + while(its.hasNext()){ + Map.Entry workEntry = its.next(); + Worker work = (Worker)workEntry.getValue(); + if(work.status == 1){ + return work; + } + } + return null; + } + + +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy new file mode 100644 index 000000000..7e81c3afd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/DataUtil.groovy @@ -0,0 +1,311 @@ +package com.fangjia.script.gj.paimai + +import org.apache.commons.codec.binary.Base64 +import org.apache.commons.lang.StringUtils +import sun.font.Font2D +import sun.font.Font2DHandle +import sun.font.TrueTypeFont +import sun.font.TrueTypeGlyphMapper + +import java.awt.Font +import java.awt.font.FontRenderContext +import java.awt.font.GlyphVector +import java.awt.geom.AffineTransform +import java.lang.reflect.Field +import java.util.regex.Matcher +import java.util.regex.Pattern + +class DataUtil { + + + static def checkMinDate(curDate, checkDate){ + if(curDate){ + def minDate = com.fangjia.script.util.DateUtils.parseFromTightByPattern(checkDate,"yyy-MM-dd") + if(curDate.getTime()maxDate.getTime()){ + return null + } + } + return curDate + } + static def getAreaNumber(def input, def maxArea) { + Pattern pattern = Pattern.compile('\\d+(.\\d+)?(平方米|㎡|m2|方米|平)') + Matcher matcher = pattern.matcher(input) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() > 0) { + def area = null + + for (def val : list) { + def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", "") + if (maxArea) { + if (v.isDouble()) { + if (v.toDouble() < maxArea) { + area = v + break + } + } + } else { + area = v + break + } + } + + if (area) { + return area + } + } + return null + } + + static def getPaiMaiType(def s) { + if (s) { + if (s.toString().contains("结束")) { + return "已完成" + } else if (s.toString().contains("进行") || s.toString().contains("正在拍卖")) { + return "进行中" + } else if (s.toString().contains("即将开始")) { + return "即将开始" + } else if (s.toString().contains("流拍")) { + return "流拍" + } else if (s.toString().contains("撤回")) { + return "撤回" + } else if (s.toString().contains("暂缓")) { + return "暂缓" + } else if (s.toString().contains("中止")) { + return "中止" + } else if (s.toString().contains("失败")) { + return "失败" + } + } + return s + } + + + static def getDateRange(def s) { + s = s.replace("起", "") + Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}[月|日]\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + def str = matcher.group() + list.add(str) +// if(str) { +// def indx1 = str.indexOf("年") +// def indx2 = str.indexOf("月") +// def month = str.substring(indx1+1, indx2); +// if(month.toInteger()<12) { +// list.add(str); +// } +// } + } + if (list.size() >= 1) { + String s1 = list.get(0) + s1 = s1.replace("法院将于", "").replace(" ", "") + def array = s1.split("至") + return array + } + return null + } + + + static def getStartPrice(def s) { + Pattern pattern = Pattern.compile('起拍价:\\d+(.\\d+)?万?元') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() >= 1) { + String s1 = list.get(0) + if (s1.contains("万元")) { + s1 = s1.toString().replace("起拍价:", "").replace("万元", "").replace(",", "").replace(",", "").replace(" ","") + return (int) (s1.toDouble() * 10000) + } else { + s1 = s1.toString().replace("起拍价:", "").replace("元", "").replace(",", "").replace(",", "").replace(" ","") + if (s1.isInteger()) { + return s1.toInteger() + } else { + return (int) (s1.toDouble()) + } + } + } + return null + } + + static def getEstateType(def s) { + Pattern pattern = Pattern.compile('房屋类型:.*;') + Matcher matcher = pattern.matcher(s) + + List list = new ArrayList() + while (matcher.find()) { + list.add(matcher.group()) + } + if (list.size() >= 1) { + String s1 = list.get(0) + if (s1) { + s1 = s1.substring(s1.indexOf(":") + 1, s1.indexOf(";")) + return s1.replace(")", "").replace("】", "") + } + } + return null + } + + static def findEstateType(def s) { + if (s) { + def villTypes = ["独栋", "联排", "联列", "叠加", "双拼", "花园面积", "泳池", "别墅", "花园住宅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"] + for (def type : villTypes) { + if (s.contains(type)) { + return "别墅" + } + } + + def districtTypes = ["住宅", "公寓"] + for (def type : districtTypes) { + if (s.contains(type)) { + return "普通住宅" + } + } + } + + return null + + } + + static def filterChar(def detailMap) { + def newDetailMap = [:] + detailMap.each { def k, v -> + if (k.toString().contains(".")) { + newDetailMap.put(k.toString().replace(".", "\\u002e"), v) + } else { + newDetailMap.put(k, v) + } + } + return newDetailMap + } + + /** + * 匹配符合条件的字符串 + * + * @param str + * @param pattern + * @return + */ + static String match(String str, Pattern pattern) { + if (str == null) { + return null + } + Matcher matcher = pattern.matcher(str) + if (matcher.find()) { + return matcher.group() + } + return null + } + + /** + * font-secret字符串专用解密工具 + * + * @param key 密匙 + * @param encodeString 加密后的字符串 + * @return 解密后的字符串 + */ + static String decodeString(String key, String encodeString) { + if (!key) { + return "" + } + try { + //base64解码,初始化字体 + Base64 base64 = new Base64() + byte[] ss = base64.decodeBase64(key) + InputStream inputStream = new ByteArrayInputStream(ss) + Font dynamicFont = Font.createFont(Font.TRUETYPE_FONT, inputStream) + FontRenderContext fontRenderContext = new FontRenderContext(new AffineTransform(), false, false) + GlyphVector glyphVector = dynamicFont.createGlyphVector(fontRenderContext, "") + + //获取font中字形的映射关系,字段为private,使用反射 + Class clazz = Font.class + Field[] fs = clazz.getDeclaredFields() + Font2DHandle font2DHandle = null + for (int i = 0; i < fs.length; i++) { + fs[i].setAccessible(true)// 将目标属性设置为可以访问 + if (fs[i].getName().equals("font2DHandle")) { + font2DHandle = (Font2DHandle) fs[i].get(dynamicFont) + } + + } + + //得到映射关系 + Font2D font2D = font2DHandle.font2D + TrueTypeFont trueTypeFont = (TrueTypeFont) font2D + TrueTypeGlyphMapper charToGlyphMapper = (TrueTypeGlyphMapper) trueTypeFont.getMapper() + + //开始解密,encodeString为加密后的字符串 + StringBuffer buffer = new StringBuffer() + char[] chars = encodeString.toCharArray() + for (int i = 0; i < chars.length; i++) { + buffer.append(charToGlyphMapper.charToGlyph(chars[i]) - 1) + } + return buffer.toString() + } catch (Exception e) { + e.printStackTrace() + } + return "" + } + + static List SITE_SUFFIXS = [".gov.cn",".com.cn", ".com", ".cn", ".net", ".org", ".cc", ".tm", ".biz", ".info", ".tv"] + + static String urlToSiteName(String url) { + if (StringUtils.isEmpty(url)) { + return null + } + + for (String suffix : SITE_SUFFIXS) { + if (url.indexOf(suffix) > -1) { + String s = StringUtils.substringBefore(url, suffix) + if (s.indexOf(".") > -1) { + s = StringUtils.substringAfterLast(s, ".") + if (StringUtils.isNotBlank(s)) { + return s + suffix + } + } else if (s.indexOf("://") > -1) { + s = StringUtils.substringAfterLast(s, "://") + if (StringUtils.isNotBlank(s)) { + return s + suffix + } + } else if (s.indexOf("://") == -1 && s.indexOf(".") == -1) { + return s + suffix + } + + return null + } + } + return null + } + + static double getRound(int scale, double dSource) { + BigDecimal deSource = new BigDecimal(dSource) + return deSource.setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue() + } + + static def filterEmpty(def s){ + if(s && s.contains("暂无")){ + return null + } + return s + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy index d60d5f02c..c3249e561 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy @@ -514,8 +514,9 @@ class GaoDeUtils { // "2a1fb75990f2b39f6221e50892dd3988" // ] + //35925efd7333f1df52145e51b8e4e56a 15021384995 def KeyList = [ - "10bcb421d2ee4e51ed9268d6dca58ad1" + "10bcb421d2ee4e51ed9268d6dca58ad1"//17740818975 ] def count = RandomUtils.nextInt(KeyList.size()) return KeyList.get(count); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy new file mode 100644 index 000000000..b432108a2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/IcbcCrawl.groovy @@ -0,0 +1,348 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import javafx.scene.input.DataFormat +import net.sf.json.JSONObject +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class IcbcCrawl { + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + + static main(args) { + def task = new IcbcCrawl(); + task.run(); + } + + def run(){ + def url = "https://gf.trade.icbc.com.cn/searchproducts/pv.jhtml?resultLevel=1&FilterObjectName=%E8%B5%84%E4%BA%A7%E5%A4%84%E7%BD%AE%E5%B9%B3%E5%8F%B0&filterObjectId=8&category_Type=3&categoryId_level=&subjectChildType=1#this" + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + driver.get(url) + driver.findElementById("category_SM310000000000000006").click() + Thread.sleep(3000) + + + def urls = getAllDetailLinks(driver) + println "共:"+urls.size() + + detailCrawling(urls, driver) + + } + + def getAllDetailLinks(driver){ + def stop = false + def urls = [] + while (!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + def eles = JQueryUtils.select(".tabContent5 .ymzs_sp", doc) + if(eles){ + for(def e : eles){ + def href = e.select(".li_2_name a").attr("href") + urls << "https://gf.trade.icbc.com.cn"+href + } + def nextPage = JQueryUtils.select("a:contains(下一页)", doc) + if(nextPage){ + driver.findElement(By.linkText("下一页")).click(); + }else{ + stop = true + } + }else{ + stop = true + } +// if(urls.size()>=10){ +// stop = true +// } + } + return urls; + } + + def detailCrawling(def urls, driver){ + def rets = [] + for(def url : urls){ + driver.get(url) + Thread.sleep(5000) + def doc = Jsoup.parse(driver.getPageSource()); + def result = parser(doc, url) + if (result) { + rets << result + println rets.size() + } + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add + rets = [] + } + } + if(rets.size()>0){ + saveData(rets) + print "完" + } + } + + def parser(doc, url){ + def result = [:] + if(doc){ + def title = JQueryUtils.selectText("#prodnameb", doc) + def id = url.substring(url.indexOf("="), url.length()) + if(title){ + + def cityStr = JQueryUtils.selectText("dl:contains(所在城市:) li", doc) + if(cityStr && cityStr.contains("...")){ + def cityEles = JQueryUtils.select("dl:contains(所在城市:) li", doc) + if(cityEles){ + cityStr = cityEles[0].attr("title") + } + } + if(cityStr && cityStr.contains("省")){ + result.province = cityStr.substring(0, cityStr.indexOf("省")) + result.city = cityStr.substring(cityStr.indexOf("省")+1, cityStr.length()) + }else{ + if(cityStr.contains("自治区")){ + result.province = cityStr.substring(0, cityStr.indexOf("自治区")+3) + result.city = cityStr.substring(cityStr.indexOf("自治区")+3, cityStr.length()) + }else{ + result.city = cityStr.substring(0, cityStr.indexOf("市")) + } + } + + if(!result.city){ + return null + } + result.title = title + def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + + result.status = JQueryUtils.selectText("#message", doc).replace("本场竞买","") + + def t1 = JQueryUtils.selectText("#pricep", doc) + if(t1 && t1.contains("原因")){ + result.statusDesc = t1.substring(t1.indexOf("原因")+3, t1.toString().length()) + } + result.type = JQueryUtils.selectText("dl:contains(类  型:) dd", doc) + def enrollCount = JQueryUtils.selectText(["#signUpNum"], doc) + if(enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def lookCount = JQueryUtils.selectText(["#visitorCount"], doc) + if(lookCount) { + result.lookCount = lookCount.toInteger() + } + + def callCount = JQueryUtils.selectText(["#remindPPNum"], doc) + if(callCount) { + result.callCount = callCount.toInteger() + } + + def curPriceStr = JQueryUtils.selectText(["#new_price_key"], doc) + if(curPriceStr) { + curPriceStr = curPriceStr.replace(",", "").replace("元", "").trim() + if(curPriceStr.isInteger()) { + result.curPrice = curPriceStr.toInteger() + }else { + result.curPrice = (int)(curPriceStr.toDouble()) + } + } + + def startPriceEles = JQueryUtils.select(["dl:contains(起始价格:) dd"], doc) + if(startPriceEles) { + def startPriceStr = startPriceEles[0].attr("title") + if(startPriceStr){ + startPriceStr = startPriceStr.replace(",", "").replace("¥","") + + if(startPriceStr.isInteger()) { + result.startPrice = startPriceStr.toInteger() + }else { + result.startPrice = (int)(startPriceStr.toDouble()) + } + } + + } + + + def addPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc) + if(addPriceEles){ + def addPriceStr = addPriceEles[0].attr("title") + if(addPriceStr){ + addPriceStr = addPriceStr.replace(",", "").replace("¥","") + if(addPriceStr) { + result.addPrice = addPriceStr.toDouble().toInteger() + } + } + } + + def evaluationPriceEles = JQueryUtils.select(["dl:contains(评估价格:) dd"], doc) + if(evaluationPriceEles){ + def evaluationPriceStr = evaluationPriceEles[0].attr("title") + if(evaluationPriceStr){ + evaluationPriceStr = evaluationPriceStr.replace(",", "").replace("¥","") + if(evaluationPriceStr.isInteger()) { + result.evaluationPrice = evaluationPriceStr.toDouble().toInteger() + }else { + result.evaluationPrice = (int)(evaluationPriceStr.toDouble()) + } + } + } + + def depositPriceEles = JQueryUtils.select(["dl:contains(加价幅度:) dd"], doc) + if(depositPriceEles){ + def v = depositPriceEles[0].attr("title") + if(v){ + v = v.replace(",", "").replace("¥","") + if(v.isInteger()) { + result.depositPrice = v.toDouble().toInteger() + }else { + result.depositPrice = (int)(v.toDouble()) + } + } + } + + result.delayWeekly = JQueryUtils.selectText(["dl:contains(延时周期:) dd div:eq(0)"], doc) + result.person = JQueryUtils.selectText(["dl:contains(联系人:) dd"], doc) + result.court = JQueryUtils.selectText("#goToCourt", doc) + def mobileEles = JQueryUtils.select("dl:contains(联系电话:) dd", doc) + if(mobileEles){ + result.mobile = mobileEles[0].attr("title") + } + + def s1 = JQueryUtils.selectText("#div_announcement_gfword", doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = PaimaiDataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时") + result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时") + } + } + + def trs = JQueryUtils.select([".rygtable table tr"], doc) + if(trs) { + def detailMap = [:] + for(def tr : trs) { + def d_tds = tr.select("td") + if(d_tds && d_tds.size()>=2) { + if(d_tds.size()==3) { + if(d_tds[2].text() && d_tds[2].text()!="/"){ + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } + + }else { + if(d_tds[1].text() && d_tds[1].text()!="/") { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + } + } + } + if(detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + if(!result.address) { + result.address = detailMap.get("拍品名称") + if(!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + def imgs =JQueryUtils.select("#scroll li img", doc) + if(imgs) { + def imgLinks = [] + for(def img : imgs) { + def href = img.attr("src") + if(href.contains("icbc.com")) { + imgLinks << "https:"+href + } + } + result.imgUrls = imgLinks + } + + + def detailStr = JQueryUtils.selectText(".detail_list", doc) + if(detailStr){ + def area = DataUtil.getAreaNumber(detailStr, 5000) + if(area) { + result.area = area.toDouble() + if(result.tradePrice) { + result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble()) + } + } + } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + result.introduce = detailStr.toString() + result.url = url + result.mDate = new Date() + result.source = "融e够" + return result; + } + + } + return null + } + + def getCallCount(id){ + def url= "https://gf.trade.icbc.com.cn/wholeSaleProd/visitorCount.jhtml?prodId="+id + def s = ABuYunUtil.doPost(url, null, null, "utf-8", true) + if(s){ + JSONObject jsonObject = JSONObject.fromObject(s); + println jsonObject + } + } + + + + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + return add + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy new file mode 100644 index 000000000..e11e01e8f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/JdPaimaiCrawl.groovy @@ -0,0 +1,540 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.apache.poi.ss.util.RegionUtil +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +import javax.sound.midi.MidiSystem +import java.util.regex.Pattern + +class JdPaimaiCrawl { + + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + + static main(args) { + def task = new JdPaimaiCrawl(); + task.run(); + } + def run(){ + def existsIds = []//findAllIds() + println "已存在:"+existsIds.size() + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def links = getList(driver) + def ids = getDetailLinks(links, driver) + println "共:"+ids.size() + parserList( existsIds,ids, driver) + + } + + def getList(driver){ + def list = [] + list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId=4&_=1564047626268" +// for(def i=1;i<=32;i++){ +// if(i==8){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=560&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=573&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=579&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=584&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=589&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=593&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=598&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=604&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=613&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=617&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=621&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=632&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=6858&_=1564047626268" +// }else if(i==5){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=142&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=148&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=164&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=199&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=224&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=239&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=248&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=258&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=264&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=274&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=275&_=1564047626268" +// }else if(i==17){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1432&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1441&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1458&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1466&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1475&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1477&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1479&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=3154&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1381&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1387&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1396&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1405&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1413&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1421&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2922&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2980&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=2983&_=1564047626268" +// }else if(i==19){ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1601&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1611&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1617&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1627&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1634&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1709&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1643&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1650&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1655&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1607&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1609&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1666&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1672&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1677&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1684&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1690&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1698&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1704&_=1564047626268" +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&cityId=1705&_=1564047626268" +// }else{ +// list << "https://auction.jd.com/getJudicatureList.html?callback=jQuery4598963&limit=40&childrenCateId=12728&provinceId="+i+"&_=1564047626268" +// } + +// } + def links = [] + for(def url : list){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + //def document = ABuYunUtil.doPost(url, null, null, "utf-8", true) + if(doc){ + def json = doc.toString().substring(doc.toString().indexOf("(")+1, doc.toString().lastIndexOf(")")) + JSONObject obj = JSONObject.fromObject(json); + if(obj && obj.total){ + def total = obj.total + def page = total/40 + def p = page.toInteger() + if(page > p){ + p = p +1 + } + + for(def i=1;i<=p;i++){ + links << url+"&page="+i + } +// if(links){ +// break; +// } + } + } + } + return links; + } + + def getDetailLinks(links, driver){ + def rets = [] + def citys = ["北京","上海","天津","重庆"] + for(def link : links){ + driver.get(link) + def doc = Jsoup.parse(driver.getPageSource()); + if(doc) { + def json = doc.toString().substring(doc.toString().indexOf("(") + 1, doc.toString().lastIndexOf(")")) + JSONObject obj = JSONObject.fromObject(json); + if (obj && obj.ls) { + for(def o : obj.ls){ + def map = [:] + if(citys.contains(o.province)){ + map.province = o.province + map.city = o.province + map.region = o.city + }else{ + map.province = o.province + map.city = o.city + } + map.id = o.id.toString() + + rets << map + } + } + } + } + return rets + } + + def parserList(existsIds,list, driver){ + def rets = [] + for(def map : list) { + if(!existsIds.contains(map.id)){ + def url = "https://paimai.jd.com/" + map.id + def doc = getDocument(url, driver) + if (doc) { + def result = parsering(doc, url, map.province, map.city, map.region) + if (result) { + rets << result + } + if (rets.size() % 100 == 0) { + def add= saveData(rets) + println DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")+" add:"+add + rets = [] + } + } + } + } + if(rets.size()>0){ + saveData(rets) + print "完" + } + + } + + def saveData(def results){ + def add = 0 + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + add++ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + return add + } + + def parsering(doc, def url, def province, def city, def region){ + def result = [:] + if(doc) { + result.proince = province + result.city = city + if(region){ + result.region = region + } + def title = JQueryUtils.selectText([".pm-name"], doc) + result.title = title + result.address = JQueryUtils.selectText([".pm-location em"], doc) + if(result.address) { + //调用获取城市 +// if(!city) { +// city = RegionData.formatCity(cityMap.name) +// } + + if(city) { + def rMap = OpenApiUtils.matchDistrict(city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + } + + } + + result.status = JQueryUtils.selectText([".pm-summary .state"], doc) + + def enrollCount = JQueryUtils.selectText([".pm-summary .times span:eq(0) em"], doc) + if(enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def lookCount = JQueryUtils.selectText([".pm-summary .times span:eq(1) em"], doc) + if(lookCount) { + result.lookCount = lookCount.toInteger() + } + + def callCount = JQueryUtils.selectText([".pm-summary .times span:eq(2) em"], doc) + if(callCount) { + result.callCount = callCount.toInteger() + } + + + def curPriceStr = JQueryUtils.selectText([".mc .current em"], doc) + if(curPriceStr) { + curPriceStr = curPriceStr.replace(",", "") + if(curPriceStr.isInteger()) { + result.curPrice = curPriceStr.toInteger() + }else { + result.curPrice = (int)(curPriceStr.toDouble()) + } + } + + def startPriceStr = JQueryUtils.selectText([".mc .base em"], doc) + if(startPriceStr) { + startPriceStr = startPriceStr.replace(",", "") + + if(startPriceStr.isInteger()) { + result.startPrice = startPriceStr.toInteger() + }else { + result.startPrice = (int)(startPriceStr.toDouble()) + } + + } + + def tradePriceStr = JQueryUtils.selectText([".mc .deal em"], doc) + if(tradePriceStr) { + tradePriceStr = tradePriceStr.replace(",", "") + if(tradePriceStr.isInteger()) { + result.tradePrice = tradePriceStr.toInteger() + }else { + result.tradePrice = (int)(tradePriceStr.toDouble()) + } + + } + + def finishDateStr = JQueryUtils.selectText([".endtime"], doc) + if(finishDateStr && finishDateStr.contains("结束时间")) { + finishDateStr = finishDateStr.substring(5, finishDateStr.indexOf(" ")) + result.finishDate = DateUtils.parseFromTightByPattern(finishDateStr, "yyyy年MM月dd日 HH:mm:ss") + } + + def lis = JQueryUtils.select(".description ul li", doc) + if(lis && lis.size()) { + for(def li : lis) { + def str = li.select("span").text() + if(str.contains("起拍")) { + def v= li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + if(v.isInteger()) { + result.startPrice = v.toDouble().toInteger() + }else { + result.startPrice = (int)(v.toDouble()) + } + + } + }else if(str.contains("评估价")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v.isInteger()) { + result.evaluationPrice = v.toDouble().toInteger() + }else { + result.evaluationPrice = (int)(v.toDouble()) + } + }else if(str.contains("加价幅度")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + result.addPrice = v.toDouble().toInteger() + } + }else if(str.contains("保证金")) { + def v = li.select("em").text().replace("¥", "").replace(",", "") + if(v) { + if(v.isInteger()) { + result.depositPrice = v.toDouble().toInteger() + }else { + result.depositPrice = (int)(v.toDouble()) + } + + } + }else if(str.contains("延时周期")) { + + result.delayWeekly = li.select("em").text().replace("¥", "").replace(",", "") + + }else if(str.contains("优先竞买权人")) { + result.preemptivePerson = li.select("em").text().replace("¥", "").replace(",", "") + } + } + } + + def trs = JQueryUtils.select([".floor-content div:eq(2) table tr"], doc) + if(trs) { + def detailMap = [:] + for(def tr : trs) { + def d_tds = tr.select("td") + if(d_tds && d_tds.size()>=2) { + if(d_tds.size()==3) { + detailMap.put(d_tds[1].text(), d_tds[2].text()) + }else { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + }else { + if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) { + detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", "")) + } + } + } + if(detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + if(!result.address) { + result.address = detailMap.get("拍品名称") + if(!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + + def imgs =JQueryUtils.select(".floor-content div:eq(2) a", doc) + if(imgs) { + def imgLinks = [] + for(def img : imgs) { + def href = img.attr("href") + if(href.contains("http")) { + imgLinks << href + }else { + imgLinks << url.substring(0, url.indexOf(":")+1)+href + } + } + result.imgUrls = imgLinks + } + + } + + //解析面积 + def article = JQueryUtils.selectText([".floor-content"], doc) + if(article) { + def area = DataUtil.getAreaNumber(article, 5000) + if(!area) { + article = JQueryUtils.selectText(["body"], doc) + if(article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + + if(area) { + result.area = area.toDouble() + if(result.tradePrice) { + result.tradeAvgPrice = (int)(result.tradePrice/area.toDouble()) + } + } + } + + def s1 = JQueryUtils.selectText(["li:contains(竞买公告) .floor-content"], doc) + if(s1) { + def start = s1.indexOf("将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("将于"), s1.indexOf("止")) + } + def dateRangs = DataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + + for(def j=0;jind+1) { + date = date +"时" + } + + pattern = "yyyy年MM月dd日HH时" + } + if(j==0) { + try{ + result.startDate = DateUtils.parseFromTightByPattern(date, pattern) + }catch(Exception e){ + + } + + }else { + if(!result.finishDate) { + result.finishDate = DateUtils.parseFromTightByPattern(date, pattern) + } + } + } + } + if(!result.startPrice) { + result.startPrice = DataUtil.getStartPrice(s1) + } + + def estateType = DataUtil.findEstateType(s1) + if(estateType) { + result.matchEstateType = estateType + } + + } + + if(!result.startDate){ + def sTime = JQueryUtils.selectText(".startTime", doc).replace("开拍时间:","") + result.startDate = DateUtils.parseFromTightByPattern(sTime, "yyyy年MM月dd日") + } + if(result.finishDate){ + result.finishDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.finishDate, "2000-01-01") + } + if(result.startDate){ + result.startDate = com.fangjia.script.gj.paimai.DataUtil.checkMaxDate(result.startDate, "2050-01-01") + } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + result.introduce = JQueryUtils.select([".floor-content div:eq(2) table"], doc).toString() + result.url = url + result.city = city + result.mDate = new Date() + result.source = "京东" + return result; + } + + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } + def findAllIds(){ + def q = new BasicDBObject() + q.put("url", Pattern.compile(".*jd.com.*")) + def dbcursor = paimaiResultColl.find(q) + def ids = [] + for(def d : dbcursor){ + def url = d.url + if(url){ + def id = url.substring(url.toString().lastIndexOf("/")+1, url.toString().length()) + ids << id + } + } + return ids; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy new file mode 100644 index 000000000..63ecf3c2a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -0,0 +1,485 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.gj.paimai.ChromeDriverPool +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +public class LianjiaCrawlByChrome { + + def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new LianjiaCrawlByChrome(); + task.run() +// task.run1() + task.addDealRegion(); + task.standardRegion() +// task.updadteDealPriceField() + } + + def updadteDealPriceField(){ + def citys = CommonDatautils.findAllCity() + def index = 0 + for(def city : citys){ + index ++ + def q = new BasicDBObject() + q.put("city", city) + q.put("hall", "0") + def count = DEAL_PRICE_COLL.count(q) + println "[${index}] "+city +" "+count + if(count && count>0){ + def udata = new BasicDBObject() + udata.put("hall", 0) + DEAL_PRICE_COLL.update(q, new BasicDBObject('$set', udata), false, true); + println "[修改]"+city +" "+ count + } + } + } + + def run1(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def cityUrls = getAllCityUrls(driver) + getLogin(driver) + + for(def cityUrl : cityUrls){ + def detailUrls = getPageUrls(driver, cityUrl) + println cityUrl +" 详情链接:"+detailUrls.size() + poolCrawl( detailUrls) + } + } + + def poolCrawl(detailUrls){ + //模拟并发 + int parallelSize = 10; + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService pool = Executors.newFixedThreadPool(parallelSize); + + for (int i = 0; i < detailUrls.size(); i++) { + final def detailUrl = detailUrls[i] + pool.execute(new Runnable() { + @Override + public void run() { + + Worker work = null; + try { + latch.await();//阻塞所有线程 + System.out.println(Thread.currentThread().getName()); + while (true) {//业务需求,一直等待直到获取到驱动 + work = ChromeDriverPool.getPool(); + if (work != null) { + break; + } + Thread.sleep(1000); + } + work.start(); + ChromeDriver driver = work.getDriver(); + def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl) + println detail.city + " " + detail.region + " " + detail.district_name + System.out.println(Thread.currentThread().getName()+":"+driver); + Thread.sleep(1000 * 10); + } catch (Exception e) { + System.out.println(e.getMessage()); + }finally { + work.shutdown(); + } + } + }); + } + latch.countDown(); + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def cityMap = getAllCityUrls(driver) + getLogin(driver) + crawlData(driver, cityMap) + + + } + + def standardRegion(){ + + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd"))) + q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) + def dbcursor = DEAL_PRICE_COLL.find(q) + for(def d : dbcursor){ + if(d.region){ + def newRegion = RegionData.formatRegion(d.city, d.region) + if(newRegion && newRegion!=d.region){ + def query = new BasicDBObject() + query.put("city", d.city) + query.put("region", d.region) + def data = new BasicDBObject() + data.put("region", newRegion) + DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) + } + } + + } + } + + def addDealRegion(){ + def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + lastMonth = lastMonth +"-01" + def q = new BasicDBObject() + q.put("region", null) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = DEAL_PRICE_COLL.find(q) + def keys = [] + for(def d : dbcursor){ + keys << ["city":d.city, "name":d.district_name] + } + for(def m : keys){ + //def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district") + def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district") + if(list && list.size()==1){ + def dist = list[0] + + def query = new BasicDBObject() + query.put("city", m.city) + query.put("region", null) + query.put("district_name", m.name) +// query.put("source", "链家在线") + query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd"))) + + def data = new BasicDBObject() + data.put("region", dist.region) + println m.city+ " "+m.name +" "+dist.region + def ds = DEAL_PRICE_COLL.count(query) + DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) + } + } + } + + def crawlData(driver, cityMap){ + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + cityMap.each{city, cityUrl-> + //def detailUrls = getPageUrls(driver, cityUrl) + def detailUrls = getPageDeail(driver, city, cityUrl) + println cityUrl +" 详情链接:"+detailUrls.size() + // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) + } + } + + def getPageUrls(driver, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + }else{ + stop = true; + break; + } + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + + def getPageDeail(driver, city, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2022-03-20", "yyyy-MM-dd") + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + parse(city, e) + }else{ + stop = true; + break; + } + } + if(page>=100){ + stop = true; + break; + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + def parse(city,e){ + try { + def title = e.select(".title a").text() + if (title) { + def detail = new BasicDBObject() + def arrs = title.split(" ") + detail.city = city + detail.district_name = arrs[0] + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + if(detail.region){ + detail.region = RegionData.formatRegion(city, detail.region) + } + + def roomStr = arrs[1] + if (roomStr) { + if (roomStr.indexOf("室") > 0) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + } + if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + } + } + + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + def price = e.select(".totalPrice .number").text() + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = e.select(".title a").attr("href") + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } + } + }catch(Exception ex){ + ex.printStackTrace() + println e.select(".title a").attr("href") + } + } + + + def findRegionByThirdPary(city, district_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("name", district_name) + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1) + def dbo = null; + for(def d : dbcursor){ + dbo = d + } + if(dbo){ + return dbo.region + } + return null; + } + def getContent(driver, detailUrl){ + driver.get(detailUrl) + def str = driver.getPageSource() + if(str && str.contains("登录")){ + getLogin(driver) + return getContent(driver, detailUrl) + } + return str; + } + def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){ + if(detailUrls){ + for(def i=0;i=4){ + detail.city = eles[0].text().replace("房产网","") + detail.region = eles[2].text().replace("二手房成交","") + detail.district_name = eles[eles.size()-1].text().replace("二手房成交","") + } + if(detail.district_name) { + def price = JQueryUtils.selectText(".dealTotalPrice i", doc) + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + def dbo = findTrade(DEAL_PRICE_COLL, detail) + if (!dbo) { + def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") + if (roomStr) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")).toInteger() + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")).toInteger() + } + detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") + detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") + def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "") + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = detailUrl + detail.c_date = new Date() + detail.m_date = new Date() + + DEAL_PRICE_COLL.save(detail) + } + } + return detail; + } + + + def findTrade(def detail){ + def q = new BasicDBObject() + q.put("city", detail.city) + q.put("region", detail.region) + q.put("district_name", detail.district_name) + q.put("area", detail.area) + q.put("total_price", detail.total_price) + q.put("trade_date", detail.trade_date) + q.put("source", detail.source) + def dbo = DEAL_PRICE_COLL.findOne(q) + return dbo + } + + def getAllCityUrls(driver){ + driver.get("https://www.lianjia.com/city/") + Thread.sleep(8000) + def cityDoc = Jsoup.parse(driver.getPageSource()); + def cityEles = JQueryUtils.select(".city_province a", cityDoc) + def cityMap = [:] + def citys= ["福州"] + if(cityEles){ + for(def e : cityEles) { + def href = e.attr("href") + if (!href.contains("fang.lianjia")){ + cityMap.put(e.text(), href) + } + } + } + return cityMap + } + + def getLogin(driver){ + driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); + WebDriverWait wait = new WebDriverWait(driver,20) + Thread.sleep(10000) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + + + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + + + elemUsername.sendKeys("15021384995"); + + elemPassword.sendKeys("12qwaszx"); + + btn.submit(); // 点击行为; + } +} + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy index f98faa3ee..90a450a66 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiDataUtil.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.paimai +import java.text.SimpleDateFormat import java.util.regex.Matcher import java.util.regex.Pattern import java.util.zip.GZIPInputStream @@ -94,32 +95,32 @@ class PaimaiDataUtil { } - public static def getDateRange(def s) { - s = s.replace("起","") - Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)'); - Matcher matcher = pattern.matcher(s); - - List list = new ArrayList(); - while (matcher.find()) { - def str = matcher.group() - list.add(str); -// if(str) { -// def indx1 = str.indexOf("年") -// def indx2 = str.indexOf("月") -// def month = str.substring(indx1+1, indx2); -// if(month.toInteger()<12) { -// list.add(str); -// } -// } - } - if(list.size()>=1) { - String s1 = list.get(0) - s1 = s1.replace("法院将于", "").replace(" ","") - def array = s1.split("至"); - return array - } - return null; - } +// public static def getDateRange(def s) { +// s = s.replace("起","") +// Pattern pattern = Pattern.compile('法院将于(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)至(\\d+年\\d{1,2}月\\d{1,2}日\\d{0,2}时?)'); +// Matcher matcher = pattern.matcher(s); +// +// List list = new ArrayList(); +// while (matcher.find()) { +// def str = matcher.group() +// list.add(str); +//// if(str) { +//// def indx1 = str.indexOf("年") +//// def indx2 = str.indexOf("月") +//// def month = str.substring(indx1+1, indx2); +//// if(month.toInteger()<12) { +//// list.add(str); +//// } +//// } +// } +// if(list.size()>=1) { +// String s1 = list.get(0) +// s1 = s1.replace("法院将于", "").replace(" ","") +// def array = s1.split("至"); +// return array +// } +// return null; +// } public static def getStartPrice(def s) { @@ -159,4 +160,30 @@ class PaimaiDataUtil { return newDetailMap; } + public static def getDateRange(def s) { + s = s.replace("起","") + Pattern pattern = Pattern.compile('法院将于(\\d+年\\d+月\\d+日\\d+时)至(\\d+年\\d+月\\d+日\\d+时)'); + Matcher matcher = pattern.matcher(s); + + List list = new ArrayList(); + while (matcher.find()) { + list.add(matcher.group()); + } + if(list.size()>=1) { + String s1 = list.get(0) + s1 = s1.replace("法院将于", "") + def array = s1.split("至"); + return array + } + return null; + } + + public static Date parseFromTightByPattern(String s, String pattern){ + if (s == null || s.length() == 0){ + return null + } + SimpleDateFormat format = new SimpleDateFormat(pattern); + return format.parse(s) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy index 44664323e..6a50bd201 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/PaimaiTest.groovy @@ -4,37 +4,273 @@ import com.alibaba.fastjson.JSON import com.fangjia.core.util.DataUtil import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject -import org.jsoup.Jsoup; -import org.jsoup.nodes.Document +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver +import java.text.SimpleDateFormat import java.util.regex.Matcher import java.util.regex.Pattern -import java.util.zip.GZIPInputStream -import java.util.zip.GZIPOutputStream -import org.apache.commons.lang.StringUtils; -import java.text.SimpleDateFormat; class PaimaiTest { + + DBCollection paimaiResultColl = MongoUtils.getCollectionByAuth20190102("fangjia", "pai_mai") + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl) + + static main(args) { def task = new PaimaiTest(); - task.run(); +// task.run(); +// task.stat() + task.updatePaimai() +// task.updateFieldPaimai() + } + + def updateFieldPaimai(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in',["重庆","重庆市"])) + def index = 0 + def dbcursor = paimai43_COLL.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + for(def obj : dbcursor) { + index++ + println index + if(obj.address && obj.address.contains("可贷款")){ + obj.put("address", obj.address.substring(0,obj.address.indexOf("可贷款"))) + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + } + } + } + + def updatePaimai(){ + def q = new BasicDBObject() + q.put("url", Pattern.compile(".*jd.com.*")) + def dbcursor = paimaiResultColl.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def index =0 + for(def d : dbcursor){ + index++ + println index + if(!d.url.contains("jd.com")){ + continue + } + + if(!d.proince.contains("重庆")){ + continue + } + + println d.url + + def query = new BasicDBObject() + query.put("url", d.url) + def dbo = paimai43_COLL.findOne(query); + + DBObject obj = new BasicDBObject(); + d.each{k,v-> + obj.put(k,v) + } + + if(dbo){ + obj.m_date = new Date() + paimai43_COLL.update(new BasicDBObject('_id', obj._id), new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimai43_COLL.save(obj) + }catch(Exception e){ + println obj.url + } + } + } + } + + def stat(){ + def q = new BasicDBObject() + q.put("city", new BasicDBObject('$in', ["重庆","重庆市"])) + def dbcursor = paimaiResultColl.find(q) + def keys = [] + for(def db : dbcursor){ + if(db.matchCity){ + def key = db.matchCity +"##"+db.matchRegion+"##"+db.matchName; + if(!keys.contains(key)){ + keys << key + } +// try{ +// def jsonObject = OpenApiClient.addressMatch("重庆", db.address, "property", false, null, null) +// if(jsonObject != null && jsonObject.precise!=-1) { +// db.matchCity = jsonObject.get("city") +// db.matchRegion = jsonObject.get("district") +// db.matchName = jsonObject.get("name") +// paimaiResultColl.update(new BasicDBObject("_id", db._id),new BasicDBObject('$set',db)) +// println db.address+" "+db.matchName +// } +// }catch(Exception e){ +// println "error "+db.address +// } + + } + } + println keys.size() } def run(){ - def url = "http://www.gpai.net/sf/item2.do?Web_Item_ID=23134" - def con = find(url); - parser(con, url); + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def url = "https://s.gpai.net/sf/search.do?at=376" + def provinces = getProvice(url, driver) + def cityUrls = getCity(provinces, driver) + getDetailUrls(cityUrls, driver) + } + def getDetailUrls(cityUrls, driver){ + for(def cMap : cityUrls){ + driver.get("https:"+cMap.url) + def doc = Jsoup.parse(driver.getPageSource()); + def total = JQueryUtils.selectText("span:contains(共有拍品) label", doc) - def parser(s, url){ - Document doc = Jsoup.parse(s); + if(total){ + def totalPage = getTotalPage(total.toInteger(), 20) + def detailDatas = [] + for(def i=1;i<=totalPage;i++){ + driver.get("https:"+cMap.url+"&page="+i) + doc = Jsoup.parse(driver.getPageSource()); + def detailEles = JQueryUtils.select(".block-wrap:contains(共有拍品) .filt-result-list li .item-tit", doc) + for(def ele : detailEles){ + def dMap = [:] + dMap.proince = cMap.proince + dMap.city = cMap.city + dMap.url = ele.select("a").attr("href") + detailDatas << dMap + } + } + + def results = parser(detailDatas, driver, cMap.proince, cMap.city) + saveData(results) + println cMap.proince+" "+cMap.city+" "+results.size() + } + } + } + + + def getTotalPage(def total, def pageSize){ + if(total){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + return 0; + } + + def getCity(datas, driver){ + def citys = ["上海","北京","天津","重庆"] + def cityList = [] + for(def data : datas){ + driver.get("https:"+data.href) + def doc = Jsoup.parse(driver.getPageSource()); + if(citys.contains(data.proince)){ + def links = JQueryUtils.select(".condition .s-subtab a", doc) + for(def ele : links) { + def cityMap = [:] + cityMap.proince = data.proince + cityMap.city = data.proince + cityMap.region =ele.text() + cityMap.url = data.href + cityList << cityMap + } + }else{ + def links = JQueryUtils.select(".condition .s-subtab a", doc) + for(def ele : links){ + if(!["乌鲁木齐市","克拉玛依市","合肥市","芜湖市","蚌埠市","淮南市","马鞍山市","淮北市","铜陵市"].contains(ele.text())){ + def cityMap = [:] + cityMap.proince = data.proince + cityMap.city = ele.text() + cityMap.url = ele.attr("href") + cityList << cityMap + } + + } + } + } + return cityList + } + + def getProvice(url, driver){ + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def datas = [] + def keys = [ "天津","上海","重庆","北京","浙江","河北","陕西","山西","新疆","辽宁","吉林","江苏","福建","江西","山东","河南","湖北","湖南","西藏","宁夏","甘肃","广东","安徽","广西","海南","四川","贵州","云南"] + if(doc){ + def eles = JQueryUtils.select(".condition a", doc) + for(def e : eles){ + def data = [:] + if(!keys.contains(e.text())){ + data.proince = e.text() + data.href = e.attr("href") + datas << data + } + } + } + return datas + } + def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc + } + + def parser(detailDatas, driver, proince, city){ + def results = [] + for(def m : detailDatas){ + // try{ + driver.get("https:"+m.url) + Thread.sleep(3000) + def doc = Jsoup.parse(driver.getPageSource()); + def obj = parsering(doc, m.url) + if(obj){ + obj.proince = proince + obj.city = city + results << obj + } +// }catch(Exception e){ +// println m.url+" 解析有问题!" +// } + + } + return results + } + + def parsering(doc, url){ def result = [:] if(doc) { - def title = JQueryUtils.selectText([".d-m-title"], doc) if(!title) { title = JQueryUtils.selectText([".DivItemName"], doc) @@ -48,6 +284,9 @@ class PaimaiTest { } result.title = title result.status = status + if(title.contains("北京市朝阳区东三环中路7号4号楼28层3201")){ + println title + } if(title) { def newStr =title @@ -76,7 +315,7 @@ class PaimaiTest { if(curPriceStr.contains("成交价:")) { def tradePriceStr = JQueryUtils.selectText([".d-m-price .price-red"], doc).replace("元","") if(tradePriceStr) { - result.tradePrice = tradePriceStr.toInteger() + result.tradePrice = tradePriceStr.toDouble().toInteger() } } } @@ -85,23 +324,28 @@ class PaimaiTest { curPriceStr = JQueryUtils.selectText(["#ItemArea .red"], doc) if(curPriceStr) { curPriceStr = curPriceStr.replace(",", "").replace("元","").replace("--","") - result.tradePrice = curPriceStr.toInteger() + result.tradePrice = curPriceStr.toDouble().toInteger() } } - def enrollCount = JQueryUtils.selectText([".peoples-infos span:eq(0) b"], doc) - if(enrollCount) { + def enrollCount = JQueryUtils.selectText([".peoples-infos span:contains(人报名) b"], doc) + if(enrollCount && enrollCount.isNumber()) { result.enrollCount = enrollCount.toInteger() } - def callCount = JQueryUtils.selectText([".peoples-infos span:eq(1) b"], doc) - if(callCount) { + def callCount = JQueryUtils.selectText([".peoples-infos span:contains(人设置提醒) b"], doc) + if(callCount && callCount.isNumber()) { result.callCount = callCount.toInteger() } - def lookCount = JQueryUtils.selectText([".peoples-infos span:eq(2) b"], doc) - if(lookCount) { - result.lookCount = lookCount.toInteger() + def lookCount = JQueryUtils.selectText([".peoples-infos span:contains(人围观) b"], doc) + if(lookCount && lookCount.isNumber()) { + try{ + result.lookCount = lookCount.toInteger() + }catch(Exception e){ + println lookCount + } + } def finishDateStr = JQueryUtils.selectText(["p:contains(拍卖竞价确认时间:)"], doc) @@ -140,7 +384,7 @@ class PaimaiTest { }else if(str.contains("保证金")) { def v = str.replace("保证金:", "").replace("元", "").replace(",", "").replace("--", "") if(v) { - result.depositPrice = v.toInteger() + result.depositPrice = v.toDouble().toInteger() } }else if(str.contains("延时周期")) { @@ -184,6 +428,47 @@ class PaimaiTest { } } } + + if(detailMap && !detailMap.get("拍品名称")){ + detailMap = [:] + for(def tr : trs) { + + if(tr.text().contains("拍品名称")){ + def td1 = tr.select("td:contains(拍品名称)+td").text() + detailMap.put("拍品名称", td1) + } + + if(tr.text().contains("权利来源")){ + def td1 = tr.select("td:contains(权利来源)+td").text() + detailMap.put("权利来源", td1) + } + + if(tr.text().contains("权证情况")){ + def td1 = tr.select("td:contains(权证情况)+td").text() + detailMap.put("权证情况", td1) + } + + if(tr.text().contains("拍品所有人")){ + def td1 = tr.select("td:contains(拍品所有人)+td").text() + detailMap.put("拍品所有人", td1) + } + + if(tr.text().contains("房屋情况")){ + def td1 = tr.select("td:contains(房屋情况)+td").text() + detailMap.put("房屋情况", td1) + } + + if(tr.text().contains("权利限制情况及瑕疵情况")){ + def td1 = tr.select("td:contains(权利限制情况及瑕疵情况)+td").text() + detailMap.put("权利限制情况及瑕疵情况", td1) + } + + if(tr.text().contains("拍品介绍")){ + def td1 = tr.select("td:contains(拍品介绍)").text() + detailMap.put("拍品介绍", td1) + } + } + } result.detailMap = detailMap if(!result.address) { result.address = detailMap.get("拍品名称") @@ -248,17 +533,18 @@ class PaimaiTest { } } - result.content = DataUtil.zip(s) + result.content = DataUtil.zip(doc.toString()) def el = JQueryUtils.select([".d-block table:eq(1)"], doc) if(!el) { el = JQueryUtils.select([".d-title:contains(标的物介绍)+div"], doc) } - result.content = DataUtil.zip(s) +// result.content = DataUtil.zip(s) result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString() result.url = url result.mDate = new Date(); + return result; } def find(url){ @@ -374,4 +660,32 @@ class PaimaiTest { } return null; } + + def saveData(def results){ + + for(def ret : results){ + def q = new BasicDBObject() + q.put("url", ret.url) + def dbo = paimaiResultColl.findOne(q); + + DBObject obj = new BasicDBObject(); + ret.each{k,v-> + obj.put(k,v) + } + obj.source = "公拍网" + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + try{ + paimaiResultColl.save(obj) + }catch(Exception e){ + println obj.url + } + + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy index 244cc0d60..a7c7dccb2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/RmfysszcCrawl.groovy @@ -1,12 +1,4 @@ -package com.fangjia.script.gj.paimai - -import com.fangjia.core.util.DataUtil -import com.fangjia.script.BaseBatch -import com.fangjia.script.BatchOwner -import com.fangjia.script.Watcher -import com.fangjia.script.crawl.JQueryUtils -import com.fangjia.script.gj.OpenApiUtils -import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.gj.paimai.PaimaiDataUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -14,66 +6,266 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import org.jsoup.Jsoup import org.openqa.selenium.By -import org.openqa.selenium.JavascriptExecutor import org.openqa.selenium.WebDriver -import org.openqa.selenium.WebElement import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.support.ui.WebDriverWait import org.openqa.selenium.support.ui.ExpectedCondition import org.openqa.selenium.support.ui.ExpectedConditions -import org.openqa.selenium.support.ui.WebDriverWait +import com.fangjia.script.gj.OpenApiUtils -import java.util.concurrent.TimeUnit import java.util.regex.Matcher import java.util.regex.Pattern - +import com.fangjia.core.util.DataUtil +import com.fangjia.script.crawl.JQueryUtils System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); + + def url = "https://www1.rmfysszc.gov.cn/projects.shtml?dh=3&gpstate=1&wsbm_slt=1" WebDriver driver = new ChromeDriver() -driver.get(url); -driver.findElementById("1").click() +run(url, driver) +//driver.get(url); +//driver.findElementById("1").click() +//driver.findElementById("search_sub").click() +// +//def provinces = getAllProvinceValue(driver.getPageSource()) +//for(def m : provinces){ +// def city_setting = driver.findElementById("city") +// //${m.province_id} +// city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() +// +////driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS); +////WebDriverWait wait = new WebDriverWait(driver,20) +////wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1")))) +// Thread.sleep(10000) +// def doc = Jsoup.parse(driver.getPageSource()); +// def citys = getAllCity(doc) +// getDetailByCity(m.province, citys, driver) +//} -def city_setting = driver.findElementById("city") -city_setting.findElement(By.xpath("//option[@value='31']")).click() -driver.findElementById("search_sub").click() -//driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS); -//WebDriverWait wait = new WebDriverWait(driver,20) -//wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.cssSelector("pagecur1")))) -Thread.sleep(10000) -def doc = Jsoup.parse(driver.getPageSource()); -def links = getLinks(doc) +def run(url, driver){ + def values = getAllCityRegionValue(url, driver) + def finishCity = ["北京市","天津市","张家口市","邯郸市","保定市","邢台市","石家庄市","唐山市","秦皇岛市"] + for(def m : values){ + if(!finishCity.contains(m.city)){ + driver.get(url); + driver.findElementById("1").click() + Thread.sleep(3000) + + def pro_setting = driver.findElementById("city") + pro_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() + Thread.sleep(3000) + + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(4000) + + def region_setting = driver.findElementById("city2") + region_setting.findElement(By.xpath("//option[@value='${m.region_id}']")).click() + + driver.findElementById("search_sub").click() + Thread.sleep(10000) + getDetailByPage(m.province, m.city, m.region, driver) + } -def eles = driver.findElementsByClassName("pagecur1") -for(def ele : eles){ - ele.click(); - Thread.sleep(10000) - doc = Jsoup.parse(driver.getPageSource()); - links.addAll(getLinks(doc)) -} -def rets = [] -for(def durl : links){ - driver.get(durl); - doc = Jsoup.parse(driver.getPageSource()); - def res = parse(durl, doc); - if(res){ - rets << res } } -saveToDB(rets); + +def getAllCityRegionValue(def url, def driver){ + driver.get(url); + driver.findElementById("1").click() + driver.findElementById("search_sub").click() + + def provinces = getAllProvinceValue(driver.getPageSource()) + def values = [] + for(def m : provinces){ + def city_setting = driver.findElementById("city") + + city_setting.findElement(By.xpath("//option[@value='${m.province_id}']")).click() + + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def citys = getAllCity(doc) + def cityRegions = getAllCityByRegion(m, citys, driver) + values.addAll(cityRegions) + //getDetailByCity(m.province, citys, driver) + } + return values +} + +def getDetailByCity(province, citys, driver){ + for(def m : citys){ + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def regions = getAllRegion(doc) + getDetailByRegion(province, m.city, regions, driver) + } +} + +def getAllCityByRegion(provinceMap, citys, driver){ + def datas = [] + for(def m : citys){ + def city_setting = driver.findElementById("city1") + city_setting.findElement(By.xpath("//option[@value='${m.city_id}']")).click() + Thread.sleep(10000) + def doc = Jsoup.parse(driver.getPageSource()); + def regions = getAllRegion(provinceMap, m, doc) + datas.addAll(regions) + } + return datas +} + +def getDetailByRegion(province, city, regions, driver){ + def links = [] + for(def r : regions){ + def city_setting = driver.findElementById("city2") + city_setting.findElement(By.xpath("//option[@value='${r.region_id}']")).click() + driver.findElementById("search_sub").click() + Thread.sleep(10000) + + def stop = false; + while(!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + links.addAll(getLinks(r.region, doc)) + def eles = JQueryUtils.select("a:contains(下一页)", doc) + if(eles){ + driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() + Thread.sleep(3000) + }else{ + stop = true; + } + } + + } + def datas = parseDatas(province, city, links, driver) + saveToDB(datas); + println r.region +" "+ links.size() +} + +def getDetailByPage(province, city, region, driver){ + def links = [] + def stop = false; + while(!stop){ + def doc = Jsoup.parse(driver.getPageSource()); + links.addAll(getLinks(doc)) + def eles = JQueryUtils.select("a:contains(下一页)", doc) + if(eles){ + driver.findElement(By.xpath("//a[contains(text(), '下一页')]")).click() + Thread.sleep(3000) + }else{ + stop = true; + } + } + + def datas = parseDatas(province, city, region, links, driver) + println province +" "+city+" "+region +" "+ links.size() + saveToDB(datas); +} +def getAllProvinceValue(doc){ + def optionsEls = JQueryUtils.select("#city option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!="" && val!="0"){ + def province = eles.text() + rets << ["province":province, "province_id":val] -def parse(url, doc){ + } + + } + return rets +} + +def getAllCity(doc){ + def optionsEls = JQueryUtils.select("#city1 option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!=""){ + def province = eles.text() + rets << ["city":province, "city_id":val] + } + + } + return rets +} + +def getAllRegion(provinceMap, cityMap,doc){ + def optionsEls = JQueryUtils.select("#city2 option", doc) + def rets = [] + for(def eles : optionsEls){ + def val = eles.attr("value") + if(val!=""){ + def region = eles.text() + def regionMap = [:] + regionMap.province = provinceMap.province + regionMap.province_id = provinceMap.province_id + regionMap.city = cityMap.city + regionMap.city_id = cityMap.city_id + regionMap.region = region + regionMap.region_id = val + rets << regionMap + } + + } + return rets +} + +def parseDatas(province, city, region,links, driver){ + def datas = [] + for(def url: links){ + def doc = getDocument(url, driver) + def result = parse(city, region, url, doc) + if(result){ + result.province = province + result.city = city + result.region = region + datas << result + } + } + return datas +} + +def getDocument(url, driver){ + def stop = false + def error = 0 + def doc = null + while(!stop){ + try{ + driver.get(url) + doc = Jsoup.parse(driver.getPageSource()); + if(doc){ + stop = true; + }else{ + error++ + Thread.sleep(5000) + } + }catch(Exception e){ + error++ + } + if(error>=10){ + stop = true; + } + } + return doc +} + +def parse(city, region, url, doc){ def result=[:] if(doc) { + def title = JQueryUtils.selectText(["#Title"], doc) result.title = title; result.address = title if(result.address){ - def rMap = OpenApiUtils.matchDistrict("上海",result.address) + def rMap = OpenApiUtils.matchDistrict(city, result.address, null, true) if(rMap) { result.matchCity = rMap.city result.matchRegion = rMap.region @@ -99,6 +291,7 @@ def parse(url, doc){ def enrollCount = JQueryUtils.selectText([".bmnumber"], doc) if(enrollCount) { + enrollCount = enrollCount.replace("人报名","") if(enrollCount.isInteger()){ result.enrollCount = enrollCount.toInteger() } @@ -107,6 +300,7 @@ def parse(url, doc){ def callCount = JQueryUtils.selectText([".sc"], doc) if(callCount) { + callCount =callCount.replace("人收藏","") if(callCount.isInteger()){ result.callCount = callCount.toInteger() } @@ -114,6 +308,7 @@ def parse(url, doc){ def lookCount = JQueryUtils.selectText([".hits"], doc) if(lookCount) { + lookCount = lookCount.replace("次围观","") if(lookCount.isInteger()){ result.lookCount = lookCount.toInteger() } @@ -123,8 +318,8 @@ def parse(url, doc){ def curPriceStr = JQueryUtils.selectText(["#price div:eq(0)"], doc) if(curPriceStr && curPriceStr.contains("起拍价")) { curPriceStr = curPriceStr.replace("起拍价", "").replace("¥","") - if(curPriceStr.contains("万元")){ - curPriceStr = curPriceStr.replace("万元","") + if(curPriceStr.contains("万")){ + curPriceStr = curPriceStr.replace("万元","").replace("万","") if(curPriceStr.isInteger()){ result.startPrice = curPriceStr.toInteger()*10000 }else{ @@ -144,6 +339,44 @@ def parse(url, doc){ } } + def tableEles = JQueryUtils.select("#bg1", doc) + if(tableEles){ + def evaluatePriceStr = tableEles.select("td:contains(值:)").text() + if(evaluatePriceStr && evaluatePriceStr.contains(":")){ + def arrs = evaluatePriceStr.split(":") + if(arrs.size()==2){ + evaluatePriceStr = arrs[1] + } + } + + def depositPriceStr = tableEles.select("td:contains(金:)").text() + if(depositPriceStr && depositPriceStr.contains(":")){ + def arrs = depositPriceStr.split(":") + if(arrs.size()==2){ + depositPriceStr = arrs[1] + } + } + if(depositPriceStr){ + depositPriceStr = depositPriceStr.replace("万元","") + if(depositPriceStr.toString().isNumber()){ + result.depositPrice = depositPriceStr.toDouble()*10000 + } + } + + def courtStr = tableEles.select("td:contains(院:)").text() + if(courtStr && courtStr.contains(":")){ + result.court = courtStr.substring(courtStr.indexOf(":")+1, courtStr.size()) + } + def personStr = tableEles.select("td:contains(人:)").text() + if(personStr && personStr.contains(":")){ + result.person = personStr.substring(personStr.indexOf(":")+1, personStr.size()) + } + + def countStr = tableEles.select("td:contains(拍卖阶段:)").text().replace("拍卖阶段:","") + if(countStr && countStr.contains("第") && countStr.contains("次")){ + result.count = countStr.substring(countStr.indexOf("第")+1, countStr.indexOf("次")) + } + } def lis = JQueryUtils.select("#bg1 div:eq(0) table tr", doc) if(lis && lis.size()) { @@ -151,18 +384,23 @@ def parse(url, doc){ def str = li.select("span").text() if(str.contains("评估值")) { def v = li.select("span:eq(1)").text().replace("万元", "") - if(v.isInteger()) { - result.evaluationPrice = v.toInteger()*10000 - }else{ - result.evaluationPrice = (int)(v.toDouble()*10000) + if(v.isNumber()){ + if(v.isInteger()) { + result.evaluationPrice = v.toInteger()*10000 + }else{ + result.evaluationPrice = (int)(v.toDouble()*10000) + } } + }else if(str.contains("保证金")) { def v = li.select("span:eq(1)").text().replace("万元", "") - if(v) { - if(v.isInteger()) { - result.depositPrice = v.toInteger()*10000 - }else { - result.depositPrice = (int)(v.toDouble()*10000) + if(v.isNumber()) { + if (v) { + if (v.isInteger()) { + result.depositPrice = v.toInteger() * 10000 + } else { + result.depositPrice = (int) (v.toDouble() * 10000) + } } } }else if (str.contains("公告日期:")){ @@ -177,13 +415,32 @@ def parse(url, doc){ def trs = tables[1].select("tr") if(trs) { def detailMap = [:] + for(def tr : trs) { def d_tds = tr.select("td") + if(tr.text().contains("调查情况表")){ + continue + } + if(d_tds.size()>1){ + def lastEleStr = d_tds[d_tds.size()-1].toString() + if(lastEleStr.contains('height="42"')){ + d_tds.remove(d_tds[d_tds.size()-1]) + } + } + + if(d_tds.size()>3){ + continue + } if(d_tds && d_tds.size()>=2) { if(d_tds.size()==3) { - detailMap.put(d_tds[1].text(), d_tds[2].text()) + if(d_tds[1].text() && d_tds[2].text()){ + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } + }else { - detailMap.put(d_tds[0].text(), d_tds[1].text()) + if(d_tds[0].text() && d_tds[1].text()){ + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } } }else { if(d_tds[0] && d_tds[0].text().contains("拍品介绍")) { @@ -202,8 +459,6 @@ def parse(url, doc){ result.introduce = tables[1].toString() } - - def imgs =JQueryUtils.select("#bdjs11 img", doc) if(imgs) { def imgLinks = [] @@ -232,6 +487,20 @@ def parse(url, doc){ result.tradeAvgPrice = (int)(result.tradePrice/result.area) } + def s1 = JQueryUtils.selectText(["#pmgg"], doc) + if(s1) { + def start = s1.indexOf("法院将于") + def end = s1.indexOf("止") + if(start>-1 && end>-1 && end>start){ + s1 = s1.substring(s1.indexOf("法院将于"), s1.indexOf("止")) + } + def dateRangs = PaimaiDataUtil.getDateRange(s1) + if(dateRangs && dateRangs.size()==2) { + result.startDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[0], "yyyy年MM月dd日HH时") + result.finishDate = PaimaiDataUtil.parseFromTightByPattern(dateRangs[1], "yyyy年MM月dd日HH时") + } + } + result.content = DataUtil.zip(doc.toString()) result.url = url @@ -246,7 +515,7 @@ def getLinks(doc){ if(eles){ for(def el :eles){ def href = el.attr("href") - urls << "https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length()) + urls <<"https://www.rmfysszc.gov.cn/statichtml/rm_obj/"+href.substring(href.indexOf("/")+1,href.length()) } } } @@ -264,13 +533,19 @@ def saveToDB(rets){ ret.each{k,v-> obj.put(k,v) } - - if(dbo){ - obj.remove("url") - paimaiResultColl.update(q, new BasicDBObject('$set', obj)) - }else{ - paimaiResultColl.save(obj) - } + obj.source= "人民法院诉讼资产网" + try{ + if(dbo){ + obj.m_date = new Date() + paimaiResultColl.update(q, new BasicDBObject('$set', obj)) + }else{ + obj.c_date = new Date() + obj.m_date = new Date() + paimaiResultColl.save(obj) + } + }catch(Exception e){ + println "error:"+obj.url + } } } @@ -287,14 +562,16 @@ def getAreaNumber(input, maxArea) { for (def val : list) { def v = val.replace("平方米", "").replace("㎡", "").replace("平", "").replace("方米", "").replace(",", "").replace("m2", ""); - if (maxArea) { - if (v.toDouble() < maxArea) { - area = v; + if(v && v.isNumber()){ + if (maxArea) { + if (v.toDouble() < maxArea) { + area = v; + break; + } + } else { + area = v break; } - } else { - area = v - break; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy index f0b328f11..68b73d58e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/TaobaoPaimaiCrawl.groovy @@ -1,7 +1,12 @@ package com.fangjia.script.gj.paimai import com.fangjia.script.crawl.JQueryUtils -import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.util.DateUtils +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver class TaobaoPaimaiCrawl { static main(args) { @@ -9,55 +14,432 @@ class TaobaoPaimaiCrawl { task.run(); } def run(){ - def seedUrl = "https://sf.taobao.com/item_list.htm?spm=a213w.7398504.filter.86.675028eabFDx9r&category=50025969&auction_source=0&city=%C8%FD%D1%C7&sorder=2&st_param=-1&auction_start_seg=-1" - def regionUrls = regionCrawl(seedUrl); - def pageUrls = pageUrlCrawl(regionUrls) - detailCrawl(pageUrls) + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); +// def url = "https://sf.taobao.com/list/50025969__-1.htm?spm=a213w.7398504.filter.62.73e954b3fMWNLd&auction_source=0&st_param=-1&auction_start_seg=-1" + def url = "https://sf.taobao.com/list/50025969__2.htm?spm=a213w.7398504.filter.65.101554b3Nynwwp&auction_source=0&st_param=-1&auction_start_seg=-1" + WebDriver driver = new ChromeDriver() + getAllCityRegions(driver, url); } - def regionCrawl(def seedUrl){ - def document = CrawalUtil.getDocumentByABuYun(seedUrl, "", "gbk"); - def regionUrls = [] - if(document){ - def eles = JQueryUtils.select(".small-subcondion a", document) - if(eles){ - for(def el : eles){ - def reiongUrl = el.attr("href").trim() - if(reiongUrl.startsWith("//")){ - regionUrls << "http:"+reiongUrl + def getAllCityRegions(driver, url){ + + def provinces = getAllProvince(driver, url) + def citys = ["天津","上海","北京","重庆"] + + for(def province : provinces){ + driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click() + def doc = Jsoup.parse(driver.getPageSource()); + def cityEles= JQueryUtils.select(".unfold .J_FilterCity a") + for(def cityEle : cityEles){ + def city = cityEle.text() + if(citys.contains(city)){ + driver.findElement(By.xpath("//a[contains(text(), '${province}')]")).click() + doc = Jsoup.parse(driver.getPageSource()); + + driver.findElement(By.cssSelector(".J_FilterCity a")).click() + doc = Jsoup.parse(driver.getPageSource()); + def regionEles= JQueryUtils.select(".J_SubCondition .small-subcondio ul li a") + for(def r : regionEles){ + def m = [:] + m.province = province + m.city = city + m.region = r.text() + def urls = crawlLinks(m, driver) + parse(m, driver, urls) } + }else{ + def m = [:] + m.province = province + m.city = city + def urls = crawlLinks(m, driver) + parse(m, driver, urls) } } } - return regionUrls; } - def pageUrlCrawl(regionUrls){ + def crawlLinks(def m, def driver){ + def stop = false def urls = [] - for(def url : regionUrls){ - def document = CrawalUtil.getDocumentByABuYun(url, "", "gbk"); - if(document){ - def pages = JQueryUtils.selectText(".page-total",document) - if(pages){ - pages = pages.toInteger() - for(def page=1;page -1) { + result.address = addr.substring(0, addr.indexOf("室") + 1) + } else { + result.address = addr + } + + if (result.city) { + if (result.address.contains(city + "市")) { + result.address = result.address.substring(result.address.indexOf(city) + city.length() + 1) + } else { + result.address = result.address.substring(result.address.indexOf(city) + city.length()) + } + } + + + if (result.address.indexOf("】") > -1) { + result.address = result.address.substring(result.address.indexOf("】") + 1) + } + + + if (result.city) { + def rMap = OpenApiUtils.matchDistrict(result.city, result.title, null, true) + if (rMap) { + result.matchCity = rMap.city + result.matchRegion = rMap.region + result.matchName = rMap.name + result.matchAddress = rMap.address + result.houseNum = rMap.house_num + result.roomNum = rMap.room_num + } + } + } + if (statusDesc) { + def status = statusDesc.substring(statusDesc.indexOf("本场拍卖") + 5, statusDesc.length() - 1) + result.status = DataUtil.getPaiMaiType(status) + result.statusDesc = statusDesc + } + + def statusEle = JQueryUtils.select(".status-tip", doc) + if (statusEle) { + def sEle = JQueryUtils.select(".status-ing", doc) + if (sEle) { + result.status = "进行中" + } else { + result.status = "即将开始" + } + } + def curPrice = JQueryUtils.selectText(["#sf-price"], doc) + if (curPrice) { + if (curPrice.contains("当前价")) { + curPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (curPrice) { + curPrice = curPrice.replace(",", "") + if (curPrice.isInteger()) { + result.curPrice = curPrice.toInteger() + } else { + result.curPrice = (int) (curPrice.toDouble()) + } + } + } else if (curPrice.contains("成交价")) { + def tradePrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (tradePrice) { + tradePrice = tradePrice.replace(",", "") + if (tradePrice.isInteger()) { + result.tradePrice = tradePrice.toInteger() + } else { + result.tradePrice = (int) (tradePrice.toDouble()) + } + + } + } else if (curPrice.contains("起拍价")) { + def startPrice = JQueryUtils.selectText(["#sf-price .J_Price em:eq(0)"], doc) + if (startPrice) { + startPrice = startPrice.replace(",", "") + if (startPrice.isInteger()) { + result.startPrice = startPrice.toInteger() + } else { + result.startPrice = (int) (startPrice.toDouble()) + } + + } + } + } + + def finishStr = JQueryUtils.selectText([".pm-bid-eyebrow .over-title"], doc) + if (finishStr) { + if (finishStr.contains("结束时间")) { + finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text() + if (finishStr) { + result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss") + } + } else if (finishStr.contains("成交时间")) { + finishStr = JQueryUtils.select([".pm-main .J_TimeLeft"], doc).text() + if (finishStr) { + result.finishDate = DateUtils.parseFromTightByPattern(finishStr, "yyyy/MM/dd HH:mm:ss") + } + } + } + + def enrollCount = JQueryUtils.selectText([".pm-remind .J_Applyer"], doc) + if (enrollCount) { + result.enrollCount = enrollCount.toInteger() + } + + def callCount = JQueryUtils.selectText(["#J_NotifyNum"], doc) + if (callCount) { + result.callCount = callCount.toInteger() + } + + def lookCount = JQueryUtils.selectText(["#J_Looker"], doc) + if (lookCount) { + result.lookCount = lookCount.toInteger() + } + def tds = JQueryUtils.select(["#J_HoverShow td"], doc) + if (tds && tds.size() > 0) { + for (def td : tds) { + def str = JQueryUtils.selectText("span:eq(0)", td).replace(" ", "") + if (str) { + if (str.contains("变卖价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.sellOffPrice = p.toInteger() + } else { + result.sellOffPrice = (int) (p.toDouble()) + } + + } + } else if (str.contains("加价幅度")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.addPrice = p.toInteger() + } else { + result.addPrice = (int) (p.toDouble()) + } + + } + } else if (str.contains("类型")) { + + result.type = td.select(".pay-type").text() + + } else if (str.contains("保证金")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.depositPrice = p.toInteger() + } else { + result.depositPrice = (int) (p.toDouble()) + } + } + } else if (str.contains("变卖周期")) { + + result.sellOffWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("竞价周期")) { + + result.pricePkWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("优先购买权")) { + + result.preemptivePerson = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("变卖预缴款")) { + def str1 = td.select("span:eq(1) .J_Price").text() + def str2 = td.select("span:eq(1) .J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.sellOffAdvancePay = p.toInteger() + } else { + result.sellOffAdvancePay = (int) (p.toDouble()) + } + } + + } else if (str.contains("评估价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.evaluationPrice = p.toInteger() + } else { + result.evaluationPrice = (int) (p.toDouble()) + } + } + } else if (str.contains("延时周期")) { + + result.delayWeekly = td.select("span:eq(1)").text().replace(": ", "") + + } else if (str.contains("起拍价")) { + def str1 = td.select(".J_Price").text() + def str2 = td.select(".J_Price .J_Price-tip").text() + def p = str1.replace(str2, "").replace(",", "") + if (p) { + if (p.isInteger()) { + result.startPrice = p.toInteger() + } else { + result.startPrice = (int) (p.toDouble()) + } + } + } + } + } + } + + result.rule = JQueryUtils.selectText(".pai-info .pai-content", doc) + + def trs = JQueryUtils.select(["#J_desc table tr"], doc) + if (trs) { + def detailMap = [:] + for (def tr : trs) { + def d_tds = tr.select("td") + if (d_tds && d_tds.size() >= 2) { + if (d_tds.size() == 3) { + detailMap.put(d_tds[1].text(), d_tds[2].text()) + } else { + detailMap.put(d_tds[0].text(), d_tds[1].text()) + } + } else { + if (d_tds[0] && d_tds[0].text().contains("拍品介绍")) { + detailMap.put("拍品介绍", d_tds[0].text().replace("拍品介绍 ", "")) + } + } + } + + if (detailMap) { + result.detailMap = DataUtil.filterChar(detailMap) + } + + if (!result.address) { + result.address = detailMap.get("拍品名称") + if (!result.address) { + result.address = detailMap.get("标的名称") + } + } + + } + + def imgs = JQueryUtils.select([".slide-bigpic img"], doc) + if (imgs) { + def imgLinks = [] + for (def img : imgs) { + def href = img.attr("src") + if (href.contains("http")) { + imgLinks << href + } else { + imgLinks << url.substring(0, url.indexOf(":") + 1) + href + } + + } + result.imgUrls = imgLinks + } + } + + //面积 + def article = JQueryUtils.selectText(["#J_ItemNotice"], doc) + if (article) { + def area = DataUtil.getAreaNumber(article, 5000) + if (!area) { + article = JQueryUtils.selectText(["#J_NoticeDetail"], doc) + if (article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + if (!area) { + article = JQueryUtils.selectText(["body"], doc) + if (article) { + area = DataUtil.getAreaNumber(article, 5000) + } + } + if (area) { + result.area = area.toDouble() + if (result.tradePrice) { + result.tradeAvgPrice = (int) (result.tradePrice / area.toDouble()) } } } - } - return urls; - } - def detailCrawl(pageUrls){ - if(pageUrls){ - for(def pageUrl : pageUrls){ - def document = CrawalUtil.getDocumentByABuYun(pageUrl, "", "gbk"); - if(document){ - def pages = JQueryUtils.select(".page-total", document) + + def s1 = JQueryUtils.selectText(["#J_NoticeDetail"], doc) + if (s1) { + def dateRangs = DataUtil.getDateRange(s1) + if (dateRangs) { + for (def j = 0; j < dateRangs.size(); j++) { + def date = dateRangs[j] + def pattern = null + if (date.endsWith("日时") || date.endsWith("日")) { + pattern = "yyyy年MM月dd日" + } else { + + def ind = date.indexOf("日") + if (date.indexOf("时") == -1 && date.toString().length() > ind + 1) { + date = date + "时" + } + + pattern = "yyyy年MM月dd日HH时" + } + if (j == 0) { + result.startDate = DateUtils.parseFromTightByPattern(date, pattern) + } else { + if (!result.finishDate) { + result.finishDate = DateUtils.parseFromTightByPattern(date, pattern) + } + } + } + } + + if (!result.startPrice) { + result.startPrice = DataUtil.getStartPrice(s1) } } + + result.content = com.fangjia.core.util.DataUtil.zip(doc.toString()) + def estateType = DataUtil.findEstateType(s) + if (estateType) { + result.matchEstateType = estateType + } + + result.introduce = JQueryUtils.select(["#J_desc table"], doc).toString() + result.url = url + result.mDate = new Date() + results << result } + return results; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy new file mode 100644 index 000000000..72d9c4559 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy @@ -0,0 +1,31 @@ +package com.fangjia.script.gj.paimai + +import org.openqa.selenium.chrome.ChromeDriver + +class Worker { + private ChromeDriver driver; + /** + * 状态修改要让其他线程可见 + * 1就绪 2运行 + */ + private volatile int status; + + public Worker(){ + driver = new ChromeDriver(); + status = 1; + } + + + public void shutdown(){ + this.status = 1; + } + + + public void start(){ + this.status = 2; + } + + public ChromeDriver getDriver() { + return driver; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy index 40b17049e..084a3d183 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/EvaluationBatch.groovy @@ -5,10 +5,15 @@ import com.fangjia.core.util.DataUtil import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.OpenApiClient +import groovy.sql.Sql + +class BlankEvaluationBatch { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -class EvaluationBatch { static main(args) { - def task = new EvaluationBatch(); + def task = new BlankEvaluationBatch(); task.run() } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy new file mode 100644 index 000000000..06218bbcf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.gj.price + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql + +class FzMatchTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new FzMatchTest(); +// task.run() + task.caluatePPE() + } + + def caluatePPE(){ + def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区_结果1).xlsx" + ExcelUtils.multiThreadingModify(filePath, { data -> + if(data['F'] && data['G']){ + def dist = CommonDatautils.findDistrict("福州", data['F'], data['G'], "district") + if(dist){ + def avg_price = findAvgPrice(dist._id.toString()) + if(data['i'] && avg_price){ + def weekendPrice = data['i'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + data['J'] = ppe15 + } + } + } + },1) + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } + + def run(){ + def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区).xlsx" + def city = "福州" + ExcelUtils.multiThreadingModify(filePath, { data -> +// if(data['G']){ +// if(!data['H']){ +// def dist = CommonDatautils.findDistrict(city, data['F'], data['G'], "district") +// +// if(dist && dist.finish_building_date){ +// data['H'] = dist.finish_building_date.substring(0,4) +// } +// } +// if(!data['I']){ +// def price = CommonDatautils.queryDistrictPrice(city, data['F'], data['G']) +// if(price){ +// data['I'] = price +// } +// } +// +// } + + if(!data['G'] && data['B']){ + def name = data['B'].replace("(", "").replace(")","").replace("\\","").replace("#","") + def rMap = SeaweedDataMatchProcess.doMatch(city, null, name, null); + if(rMap) { + data['F'] = rMap.region + data['G'] = rMap.name + def dist = CommonDatautils.findDistrictById(rMap.id) + if(dist && dist.finish_building_date){ + data['H'] = dist.finish_building_date.substring(0,4) + } + + def price = CommonDatautils.queryDistrictPrice(city, rMap.region, rMap.name) + if(price){ + data['I'] = price + } + } + } + }, 1); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index 5be06fa7f..0470fa957 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -37,7 +37,10 @@ class CrawlDistrictIdSelect { } def run(){ - def datas = findCrawlData() + def source = "58同城" + def root = "58.com" + def shortCityMap = findCitySeeds(root) + def datas = findCrawlData(source, shortCityMap, root) def list = [] datas.each {k,v-> if(v){ @@ -58,15 +61,32 @@ class CrawlDistrictIdSelect { } } - def findCrawlData(){ + def findCitySeeds(root){ + def q = new BasicDBObject() + q.put("root", root) + q.put("city", "福州") + + def dbcursor = districtseedsColl.find(q) + def cityMap = [:] + for(def d : dbcursor){ + if(!cityMap.keySet().contains(d.city)){ + cityMap.put(d.city, d.city_short) + } + } + return cityMap + + } + + def findCrawlData(def source, shortCityMap, def root){ def q = new BasicDBObject() q.put("city","福州") - q.put("source", "城市房产") - q.put("s_date",202012) + q.put("source", source) + q.put("s_date",202112) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = ".cityhouse.cn/ha/" + def prefix = "xiaoqu/" def keyList =new HashSet() + def index = 0 for(def db : dbcursor){ def url = null if(!db.url){ @@ -76,33 +96,42 @@ class CrawlDistrictIdSelect { }else{ url = db.url } - if(url && url.contains(prefix) ){ - def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html")) + + if(url){ + def district_id = "" + if(url.contains(prefix)){ + district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1) + }else{ + district_id = url; + } if(district_id=="0"){ continue } - def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix)) - def key = shortCity +"#"+district_id - def ids = retMap.get(key) + def ids = retMap.get(db.city) if(!ids){ ids = [] - retMap.put(key, ids) + retMap.put(db.city, ids) } - - if(!keyList.contains(key)){ - def map = new BasicDBObject() - map.root = "cityhouse.com" - map.city_short= shortCity - map.city = db.city - map.region = db.region - map.district_id = district_id - map.source_url = url - ids << map - - keyList << key + if(!keyList.contains(db.city+"#"+district_id)){ + def count = findIsExists(db.city, district_id, root) + if(!count){ + def map = new BasicDBObject() + map.root = "58.com" + map.city_short= shortCityMap.get(db.city) + map.city = db.city + map.region = db.region + map.district_id = district_id + map.source_url = url + map.name = db.name + ids << map + }else{ + index++ + println "[${index}]已存在:"+db.city +" "+db.name + } + keyList << db.city+"#"+district_id }else{ - println "重复:"+key + println "重复:"+db.city+"#"+db.name } }else{ println url +" 链接不合要求" @@ -110,4 +139,13 @@ class CrawlDistrictIdSelect { } return retMap } + + def findIsExists(city, district_id, root){ + def q = new BasicDBObject() + q.put("city",city) + q.put("district_id", district_id) + q.put("root",root) + def count = districtseedsColl.count(q) + return count + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy index 0f65998a6..8a490f49d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy @@ -68,7 +68,7 @@ class FZloationUpdate { Long lng0 = lng*10000000000L udata.put("lat", lat0) udata.put("lng", lng0 ) - DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + // DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag } @@ -153,7 +153,7 @@ class FZloationUpdate { if(addr){ def rMap = [:] def reps = GaoDeApiUtils.geo(dist.city, addr); - if(reps && reps.count.toInteger()>=1){ + if(reps && reps.count && reps.count.toInteger()>=1){ def map = reps.geocodes.get(0) def location = map.location def level = map.level diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 91498750a..728e6510c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.test import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject @@ -29,25 +30,33 @@ import java.text.SimpleDateFormat // } //}) -def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); +//def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); +// +//def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; +//def countMap = [:] +//onLineSqlInstance.eachRow(sql,{r-> +// def addTime = r.visit_time +// def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") +// def ips = countMap.get(month) +// if(!ips){ +// ips = new HashSet(); +// countMap.put(month, ips) +// } +// ips << r.ip +//}); +// +//countMap.each {k,v-> +// println k+" "+v.size() +//} -def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; -def countMap = [:] -onLineSqlInstance.eachRow(sql,{r-> - def addTime = r.visit_time - def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") - def ips = countMap.get(month) - if(!ips){ - ips = new HashSet(); - countMap.put(month, ips) - } - ips << r.ip -}); -countMap.each {k,v-> - println k+" "+v.size() +def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"]) +def rets = [] +for(def d : dists){ + def year = CommonUtils.getDistrictAge(d) + rets << [d.city,d.region, d.name, year] } - +ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy index 8d971ee8a..926581423 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZDataBatch.groovy @@ -11,14 +11,17 @@ import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.rabbitmq.client.AMQP import net.sf.json.JSONObject import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.bson.types.ObjectId import java.text.DecimalFormat +import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.TimeUnit +import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern @@ -34,6 +37,9 @@ class CZDataBatch extends BaseBatch{ DBCollection DIST_COLL = MongoUtils.getCollectionByName("business", "jn_bank_district", "mongo.db.fangjia.com") //final def czIdMap = readCZDistrict("d:\\常州数据结果\\原始数据\\常州江南银行住宅编号.xls"); //final def czShopMap = readCZShop("d:\\常州数据结果\\原始数据\\常州江南银行商铺--更新ing(8).xls"); + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + static main(args) { runBatch(new CZDataBatch(), args); } @@ -41,11 +47,17 @@ class CZDataBatch extends BaseBatch{ @Override void process(Object args) { def citys = CommonDatautils.findAllCity(); + citys.remove("常州"); citys.add(0, "常州") for(def city : citys){ + + def weekend = DateUtils.getFirstDayOfWeek(new Date()) + def statMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, weekend) + def commids = findAllJnBankDatas(city) + def districts = converList(CommonDatautils.findAllDistricts(city, "district", fields)); - districtRun(districts); + districtRun(districts, statMap, commids); def shops = CommonDatautils.findAllDistricts(city, "shop", fields); shops = mergeShop(shops); @@ -112,29 +124,48 @@ class CZDataBatch extends BaseBatch{ return rMap; } - def districtRun(def districts){ - + def districtRun(def districts, def statMap, commids){ def total = districts.size(); - def index = 0; - ExecutorService executorService = Executors.newFixedThreadPool(20) - + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(total) for(def i=0;i vector = sftp.ls(directory); + if (vector.size() == 1) { // 文件,直接删除 + sftp.rm(directory); + } else if (vector.size() == 2) { // 空文件夹,直接删除 + sftp.rmdir(directory); + } else { + String fileName = ""; + // 删除文件夹下所有文件 + for (ChannelSftp.LsEntry en : vector) { + fileName = en.getFilename(); + if (".".equals(fileName) || "..".equals(fileName)) { + continue; + } else { + deleteSFTP(directory + "/" + fileName); + } + } + // 删除文件夹 + sftp.rmdir(directory); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + /** * 上传文件 * @param directory 上传的目录 @@ -132,7 +181,7 @@ class SFTP { sftp.cd(filePath.toString()); } else { // 建立目录 - sftp.mkdir(filePath.toString()); + sftp.mkdir(filePath.toString()) // 进入并设置为当前目录 sftp.cd(filePath.toString()); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy index 21f508609..f43bda14e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy @@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "G00jia123456"); + email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com");//发送方 email.setSubject(mail.subject); //标题 @@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "G00jia123456"); + email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com"); email.setSubject(mail.subject); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy index c14e1d3ba..517b6b634 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy @@ -120,6 +120,9 @@ class AddressUtil { if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')) { return formatAddress } + if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷)')) { + return formatAddress + } return null } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy new file mode 100644 index 000000000..172cc24b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcChromeCrawl.groovy @@ -0,0 +1,213 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.ChromeDriver + +class CqfdcChromeCrawl { + + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) + + + static main(args) { +// def task = new CqfdcChromeCrawl(); +// task.run() + + def name = "E3号楼" //69幢(规划C-7号楼) 规划5-5号楼 + def s = DataHandleUtils.findMatchPattern(name, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + println s + + def unitNumStr = DataHandleUtils.findMatchPattern(name, "\\d+单元") + println unitNumStr + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def list = getListDatas(driver) + getDetails(driver, list); + + } + + + + def getDetails(driver, list){ + def index = 0 + for(def map_old : list){ + def datas = [] + index++ + def url = "http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=${map_old.building_id}&block=${map_old.house_num_desc}" +// driver.get(url) +// def doc = driver.getPageSource() + def cityDoc = getDocment(url, driver) + def trs = JQueryUtils.select(".building_table tr", cityDoc) + def unit1 = null + def unit2 = null + def unit3 = null + def unit1_text = null + def unit2_text = null + def unit3_text = null + for(def i=0;i + map.put(k,v) + } + datas << map + } + } + } + } + saveDatas(datas) + println index+" "+map_old.name+" "+map_old.house_num_desc +" "+datas.size() + } + } + + def getDocment(def url, def driver){ + def stop = false + def doc = null + def error = 0 + while (!stop || error>=10){ + try{ + driver.get(url) + def docment = driver.getPageSource() + if(docment){ + doc = Jsoup.parse(docment) + stop = true; + }else{ + error ++ + } + }catch(Exception e){ + error ++ + } + } + return doc; + } + + def getListDatas(driver){ + driver.get("http://cqfdc.cq315house.com/HtmlPage/PresaleDetail.html") + def datas = [] + for(def i=2;i<=208;i++){ + def docment = Jsoup.parse(driver.getPageSource()); + def trs = JQueryUtils.select("#searchbody tr", docment) + for(def tr : trs){ + def tds = tr.select("td") + + def hrefs = tds.get(3).select("a") + for(def href : hrefs){ + def map = [:] + map.house_num_desc = href.text() + def str = href.attr("href") + def building_id= str.substring(str.lastIndexOf("['")+2, str.lastIndexOf("','")) + map.building_id = building_id + + map.city = "重庆" + map.name = tds.get(0).text() + map.company = tds.get(1).text() + map.license_key = tds.get(2).text() + datas << map + } + } + println i +" "+datas.size() + try{ + driver.findElement(By.id("nextpage")).click() + }catch(Exception e){ + Thread.sleep(8000) + driver.findElement(By.id("nextpage")).click() + } + + } + println "列表:"+datas.size() + return datas; + } + + + def saveDatas(def datas){ + def rows = [] + def keys = [] + for(def d : datas){ + def q = new BasicDBObject() + q.put("city", d.city) + q.put("name",d.name) + q.put("house_num_desc", d.house_num_desc) + q.put("room_num", d.room_num) + q.put("unit_num",d.unit_num) + def obj = loudongConn.findOne(q) + if(!obj){ + def key = d.house_num_desc +"#"+d.unit_num+"#"+d.room_num + if(!keys.contains(key)){ + def udata = new BasicDBObject() + d.each {k,v-> + udata.put(k,v) + } + udata.put("c_date", new Date()) + rows << udata + keys << key + } + } + } + if(rows){ + loudongConn.insert(rows); + println "save: "+rows.size() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy new file mode 100644 index 000000000..2a231a42c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/CqfdcMatch.groovy @@ -0,0 +1,736 @@ +package com.fangjia.script.loudong + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Matcher +import java.util.regex.Pattern + +class CqfdcMatch { + + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection HOUSE_LOUDONG_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + + static DBCollection loudongFdcConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong_cqfdc", "192.168.0.235", 27018) + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + + public static final DBCollection LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia_base", "loudong", lineUrl) + public static final DBCollection HOUSE_LOUDONG_LINE_COLL = MongoUtils.getCollectionByName2("fangjia", "house_loudong", lineUrl) + + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new CqfdcMatch(); +// task.run() +// task.stat() +// task.export2() +// task.matchLoudong() + +// task.addLoudong() +// task.addFdcLoudong() + task.updateHouseLoudong() + } + + def updateHouseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + + def keys = new BasicDBObject() + keys.put("room_num", 1) + keys.put("house_floor", 1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def index = 0 + for(def d : dbcursor){ + index++ + def room_num = d.room_num + def house_floor = d.house_floor + try{ + if(room_num && house_floor && room_num.isNumber()){ + if(house_floor.toInteger()>0){ + if(!room_num.toString().startsWith(house_floor.toString()) && room_num.toInteger()<=10){ + def newValue = house_floor +"-"+room_num + println "[${index}] "+room_num +" "+house_floor+" "+newValue + def udata = new BasicDBObject() + udata.put("room_num",newValue) + HOUSE_LOUDONG_COLL.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata), false, false); + } + } + } + }catch(Exception e){ + println d._id.toString() + } + + } + } + def export(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', false)) + + def field = new BasicDBObject() + field.put("city",1) + field.put("region",1) + field.put("district_name",1) + field.put("address",1) + field.put("type",1) + def dbcursor = loudongConn.find(q, field).batchSize(5000) + def keys = [] + def rets = [] + rets << ["城市","区域","名称","地址","类型"] + for(def d : dbcursor){ + def key = d.city +"##"+d.district_name + if(!keys.contains(key)){ + keys << key + rets << [d.city, d.region, d.district_name, d.address, d.type] + } + } + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配1.xlsx",rets, "数据") + } + + def run(){ + def datas= findAddress2() + def results = match(datas); + update(results) + } + + + def export2(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', false)) + + def field = new BasicDBObject() + field.put("city",1) + field.put("name",1) + field.put("use_type",1) + def dbcursor = loudongFdcConn.find(q, field).batchSize(5000) + def keys = [] + def rets = [] + rets << ["城市","地址","类型"] + for(def d : dbcursor){ + def key = d.city +"##"+d.name + if(!keys.contains(key)){ + keys << key + rets << [d.city, d.name,d.use_type] + } + } + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆楼栋未匹配2.xlsx",rets, "数据") + } + + def update(results){ + for(def m : results){ + def q = new BasicDBObject() + q.put("city", m.old_city) + q.put("district_name", m.old_name) + q.put("region",m.old_region) + + def udata = new BasicDBObject() + udata.put("match_city", m.match_city) + udata.put("match_region", m.match_region) + udata.put("match_name", m.match_name) + loudongConn.update(q, new BasicDBObject('$set',udata),false, true) + } + } + + def match(def datas){ + def rets = [] + for(def d : datas){ + def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null) + if(jsonObject != null && jsonObject.precise!=-1) { + def map = [:] + map.match_city = jsonObject.get("city") + map.match_region = jsonObject.get("district") + map.match_name = jsonObject.get("name") + map.old_name = d.name + map.old_city = d.city + map.old_region = d.region + rets << map + } + } + return rets; + } + + def findAddress(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q) + def rets = [] + def keys = [] + for(def d : dbcursor){ + if(!keys.contains(d.name)){ + + keys << d.name + def map = [:] + map.city = d.city + map.name = d.name + rets << map + } + } + return rets; + } + + def findAddress2(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000) + def rets = [] + def keys = [] + for(def d : dbcursor){ + def key = d.city +"##"+d.region+"##"+d.district_name + if(!keys.contains(key)){ + keys << key + def map = [:] + map.city = d.city + map.region = d.region + map.name = d.district_name + rets << map + } + } + return rets; + } + + def addHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def keys = [] + def index = 0 + for(def d : dbcursor) { + index++ + println index + if(d.room_num && (d.room_num.contains("商") || d.room_num.contains("社区用房"))){ + continue + } + if(d.area.toDouble()<=25){ + continue + } + def newRegion = RegionData.formatRegion(d.city, d.region) + if (newRegion) { + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if (houseNumStr) { + house_num = houseNumStr.replace("号楼", "").replace("幢", "").replace("栋", "").replace("号", "") + } + if (house_num) { + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if (unitNum) { + unitNum = unitNum.replace("单元", "") + } + def key = d.city + "##" + newRegion + "##" + d.district_name + "##" + house_num+ "##" + unitNum +"##"+d.room_num + if(!keys.contains(key) && d.room_num) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.district_name, house_num, unitNum, d.room_num) + if(!count){ + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("house_num_unit", unitNum) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "房谱网") + if(d.floor_heigth && d.floor_heigth.isNumber()){ + udata.put("house_height", d.floor_heigth.toInteger()) + } + if(d.floor && d.floor.isNumber()){ + udata.put("house_floor", d.floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findHouseLoudongDatas(city, region, district_name, house_num, unitNum, room_num){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", district_name) + q.put("house_num", house_num) + if(unitNum){ + q.put("unit_num", unitNum) + } + q.put("room_num", room_num) + def count = HOUSE_LOUDONG_LINE_COLL.count(q) + return count; + } + + def findAllLoundongByHouseNum(){ + def q = new BasicDBObject() + def dbcursor = loudongConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(5000) + def keys = [] + for(def d : dbcursor){ + def newRegion = RegionData.formatRegion(d.city, d.region) + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def unitNum = DataHandleUtils.findMatchPattern(d.house_num, "\\d+单元") + if(unitNum){ + unitNum = unitNum.replace("单元","") + } + def key = d.city +"##"+newRegion+"##"+d.district_name+"##"+house_num + if(unitNum){ + key = key +"##"+unitNum + } + if(!keys.contains(key)){ + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.district_name, house_num, d.unit_num) + println d.district_name+" "+d.house_num+"==${house_num}==${unitNum}" + if(obj){ + def udata = new BasicDBObject() + def areas = obj.areas + if(!areas.contains(d.area)){ + areas << d.area + + def forms = obj.from + if(!forms.contains("房谱网")){ + forms << "房谱网" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + }else{ + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.district_name) + udata.put("house_num", house_num) + if(unitNum){ + udata.put("unit_num", unitNum) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("areas",[d.area]) + udata.put("from",["房谱网"]) + if(d.floor_heigth && d.floor_heigth.toString().toInteger()){ + udata.put("floors", d.floor_heigth.toString().toInteger()) + } + + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } + + } + } + } + + def addFdcHouseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT).batchSize(2000) + def keys = [] + def index = 0 + for(def d : dbcursor){ + index++ + println index + if(!d.use_type){ + continue + } + if(!d.use_type.contains("住宅")){ + continue + } + def newRegion = d.match_region + if(newRegion){ + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num+"##"+d.unit_num+"##"+d.room_num + if(!keys.contains(key)) { + keys << key + def count = findHouseLoudongDatas(d.city, newRegion, d.name, house_num, d.unit_num, d.room_num) + if(!count){ + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + def dist = null + if(d.match_city){ + dist = CommonDatautils.findDistrict(d.match_city, d.match_region, d.match_name,"district") + } + def floor = null + if(d.room_num && d.room_num.contains("-")) { + def arrs = d.room_num.split("-") + floor = arrs[0] + } + Integer room = null + Integer hall = null + if(d.room_type && d.room_type.contains("室")){ + def roomStr = d.room_type.substring(0,d.room_type.indexOf("室")) + if(roomStr){ + roomStr = roomStr.replace("一","1").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + room = roomStr.toInteger() + } + + if(d.room_type.contains("厅")){ + def hallStr = d.room_type.substring(d.room_type.indexOf("室")+1, d.room_type.indexOf("厅")) + if(hallStr){ + hallStr = hallStr.replace("一","1").replace("两","2").replace("二","2").replace("三","3").replace("四","4").replace("五","5").replace("六","6").replace("七","7").replace("八","8").replace("九","9").replace("十","10") + hall = hallStr.toInteger() + } + } + } + + + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if(d.unit_num){ + udata.put("house_num_unit", d.unit_num) + } + udata.put("room_num", d.room_num) + udata.put("house_area",d.area.toDouble()) + udata.put("status", 0) + udata.put("from", "重庆网上房地产") + udata.put("house_height", maxHeight) + if(room){ + udata.put("room", room) + } + if(hall){ + udata.put("hall", hall) + } + if(floor && floor.isNumber()){ + udata.put("house_floor", floor.toInteger()) + } + if(dist){ + udata.put("district_id", dist._id.toString()) + udata.put("name_in_seaweed", dist.name) + } + if(d.address){ + def address = AddressUtil.formatAddress(d.city, d.address) + if(address){ + udata.put("address", address) + } + } + udata.put("c_date", new Date()) + udata.put("m_date", new Date()) + HOUSE_LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + public String getMatchString(String text,String pattern){ + Pattern p=Pattern.compile(pattern); + Matcher m=p.matcher(text); + if(m.find()){ + return m.group(0) + } + return null; + } + + def addFdcBaseLoudong(){ + def q = new BasicDBObject() + def dbcursor = loudongFdcConn.find(q).batchSize(5000) + def keys = [] + for(def d : dbcursor){ + if(!d.match_region){ + continue + } + def newRegion = d.match_region + + def houseNumStr = DataHandleUtils.findMatchPattern(d.house_num_desc, "[\\w+]?-?\\w+[#|幢|栋|号]楼?.*?") + def house_num = null + if(houseNumStr){ + house_num = houseNumStr.replace("号楼","").replace("幢","").replace("栋","").replace("号","") + } + if(house_num){ + def key = d.city +"##"+newRegion+"##"+d.name+"##"+house_num + if(d.unit_num) { + key = key + "##" + d.unit_num + + if (!keys.contains(key)) { + keys << key + def obj = findLoudongOnLine(d.city, newRegion, d.name, house_num, d.unit_num) + println d.name + " " + d.house_num_desc + "==${house_num}==${d.unit_num}" + + def maxHeight = findMaxHeight(d.city, d.name, d.house_num_desc, d.unit_num) + if (obj) { + + def areas = obj.areas + if (!areas.contains(d.area)) { + areas << d.area + + def forms = obj.from + if (!forms.contains("重庆网上房地产")) { + forms << "重庆网上房地产" + } + obj.m_date = new Date() + def query = new BasicDBObject("_id", obj._id) + LOUDONG_LINE_COLL.update(query, new BasicDBObject('$set', obj)) + + } + } else { + def udata = new BasicDBObject() + udata.put("city", d.city) + udata.put("region", newRegion) + udata.put("name", d.name) + udata.put("house_num", house_num) + if (d.unit_num) { + udata.put("unit_num", d.unit_num) + } + if (d.address) { + def address = AddressUtil.formatAddress(d.city, d.address) + if (address) { + udata.put("address", address) + } + } + udata.put("areas", [d.area]) + udata.put("from", ["重庆网上房地产"]) + udata.put("floors", maxHeight) + udata.put("c_date", new Date()) + udata.put("status", 1) + LOUDONG_LINE_COLL.save(udata) + } + } + } + } + } + } + + def findMaxHeight(city, name,house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("name",name) + q.put("house_num_desc",house_num) + q.put("unit_num",unit_num) + def dbcursor = loudongFdcConn.find(q) + def heights = [] + for(def h : dbcursor){ + if(h.nominal_floor && h.nominal_floor.isNumber()){ + def height = h.nominal_floor.toInteger() + if(!heights.contains(height)){ + heights << height + } + } + } + if(heights){ + heights = heights.sort() + return heights[heights.size()-1] + } + return null + } + + def findLoudongOnLine(city, region, name, house_num, unit_num){ + def q = new BasicDBObject() + q.put("city",city) + q.put("region",region) + q.put("name",name) + q.put("house_num",house_num) + if(unit_num){ + q.put("unit_num",unit_num) + } + def dbo = LOUDONG_LINE_COLL.findOne(q) + return dbo + } + + + + def stat(){ + def q = new BasicDBObject() + q.put("match_city", new BasicDBObject('$exists', true)) + + def field = new BasicDBObject() + field.put("match_city",1) + field.put("match_region",1) + field.put("match_name",1) + def dbcursor = loudongConn.find(q, field).batchSize(2000) + def keys = [] + for(def d : dbcursor){ + def key = d.match_city+"##"+d.match_region+"##"+d.match_name + if(!keys.contains(key)){ + keys << key + } + } + + def dbcursor2 = loudongFdcConn.find(q, field).batchSize(5000) + for(def d : dbcursor2){ + def key = d.match_city+"##"+d.match_region+"##"+d.match_name + if(!keys.contains(key)){ + keys << key + } + } + println "小区:"+keys.size() + + } + + def matchLoudong(){ + def names = findAllLoudong("重庆") + print names.size() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(names.size()) + + def retMap = new ConcurrentHashMap() + for(def i=0;i + def arrs = key.split("#") + + def q = new BasicDBObject("city", arrs[0]) + q.put('region', arrs[1]); + q.put('name', arrs[2]); + + def udata = new BasicDBObject() + udata.put("district_id", m.id.toString()) + udata.put("name_in_seaweed", m.name) + udata.put("district_no", m.sid) + HOUSE_LOUDONG_COLL.update(q, new BasicDBObject('$set', udata), false, true); + index++ + println index + } + } + + def findAllLoudong(city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district_id",new BasicDBObject('$exists', false)) + + def keys = new BasicDBObject() + keys.put("city", 1) + keys.put("region", 1) + keys.put("name", 1) + def dbcursor = HOUSE_LOUDONG_COLL.find(q).batchSize(5000) + def names = new CopyOnWriteArrayList() + for(def db : dbcursor){ + def key = db.city +"#"+db.region+"#"+db.name + if(!names.contains(key)){ + names << key + } + } + return names; + } + + def addLoudong(){ +// def stat = statBaseLoudong() +// println stat +// findAllLoundongByHouseNum(); + def stat1 = statHouseLoudong() + println stat1 +// addHouseLoudong() + } + + def addFdcLoudong(){ +// addFdcBaseLoudong() + addFdcHouseLoudong() + } + + def statHouseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = HOUSE_LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.district_id && !ids.contains(d.district_id)){ + ids << d.district_id + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map + } + + def statBaseLoudong(){ + def q = new BasicDBObject() + q.put("city", "重庆") + def dbcursor = LOUDONG_LINE_COLL.find(q) + def total = dbcursor.size() + def ids = [] + for(def d : dbcursor){ + if(d.id_in_seaweed && !ids.contains(d.id_in_seaweed)){ + ids << d.id_in_seaweed + } + } + def map = [:] + map.rows_total = total + map.dist_count = ids.size() + return map + } + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy new file mode 100644 index 000000000..8684414c8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy @@ -0,0 +1,261 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.interactions.Actions +import org.openqa.selenium.support.ui.ExpectedCondition +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait + +class HouseBookChromeCrawl { + + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + static def city = "重庆" + static main(args) { + def task = new HouseBookChromeCrawl(); + task.run() + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def listPageUrl = getIndexPage(driver) + def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl) +// keywords.remove("城南家园") +// keywords.remove("康居西城") + + println city +"共 "+keywords.size()+" 个小区" + getDistrictIndex(driver, keywords) + + } + + def getListPage(driver, url){ + driver.get(url) + + def cityDoc = Jsoup.parse(driver.getPageSource()) + def total = JQueryUtils.selectText(".result-count-number:eq(0)", cityDoc) + if(total){ + def page = getTotalPage(total, 6) + def keywords = getAllDist(driver, url, page); + return keywords + } + return null; + } + + def getDistrictIndex(driver, keywords){ + for(def word : keywords){ + def code = URLEncoder.encode(URLEncoder.encode(word, "utf-8"),"utf-8") + goToDetailIndex(driver, code) + + } + } + + def getAllDist(driver, url, page){ + def keywords = [] + for(def i=73;i<=page;i++){ + driver.get(url+"?selectPages="+i) + def doc = Jsoup.parse(driver.getPageSource()) + def eles = JQueryUtils.select(".title-area .title-text", doc) + for(def e : eles){ + def text = e.text() +// if(text=="富州北郡"){ +// println i +// } + keywords << text + } + } + return keywords + } + + def getTotalPage(def total, def pageSize){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + + + def goToDetailIndex(driver, code){ + def url = "http://www.house-book.com.cn/buildAction!pnameHome?pname="+code+"&price=&totalPrice=&bsqar=&htype=&btype=&sellKind=" + driver.get(url) + + WebDriverWait wait = new WebDriverWait(driver,20) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("main")))); + def docment = Jsoup.parse(driver.getPageSource()); + def dist_name = JQueryUtils.select(".info-pname", docment).text() + def addInfo = JQueryUtils.select(".addr-info", docment).text() + def region = null + if(addInfo.contains(":") && addInfo.contains("-")){ + region = addInfo.substring(addInfo.indexOf(":")+1, addInfo.indexOf("-")) + } + + def address = JQueryUtils.select(".addr-info span", docment).text() + +// wait.until(new ExpectedCondition(){ +// @Override +// public WebElement apply(WebDriver d){ +// return d.findElement(By.linkText("[更多]")) +// } +// }).click() + driver.findElement(By.linkText("[更多]")).click() +// Thread.sleep(10) +// wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.id("shouUl")))); + def lis = checkElement(driver, "#shouUl .dongli"); + + def datas = [] + if(lis){ + for(def li : lis){ + def id = li.attr("id") + def houseNum = li.select(".title-span").text() + def map = [:] + map.id = id + map.city = city + map.region = region + map.district_name = dist_name + map.address = address + map.house_num = houseNum + map.num = li.select(".dong_ul2 li:contains(许可证号:)").text().replace("许可证号:","") + map.type = li.select(".dong_ul2 .title-span-hbType").text() + def floor_heigth = li.select(".dong_ul2 label:contains(总楼层:)").text().replace("总楼层:","").replace("层","") + if(floor_heigth){ + map.floor_heigth = floor_heigth + } + def room_count = li.select(".dong_ul2 label:contains(总户数:)").text() + if(room_count){ + room_count = room_count.substring(room_count.indexOf("总户数:")+4, room_count.lastIndexOf("户")) + if(room_count){ + map.household_count = room_count + } + } + datas << map + } + } + + println dist_name +" 共 "+datas.size()+" 栋" + for(def m : datas){ + try{ + driver.findElement(By.id(m.id)).click() + + def rows = parseDetail(driver) + println "栋divId: "+m.id +" 室:"+rows.size() + saveDate(m, rows) + }catch(Exception e){ + println dist_name+" 栋divId: "+m.id+" 找不到相应的div element" + } + + // driver.navigate().back(); + } + } + + def checkElement(driver, className){ + def loop = 0 + def stop = false + def lis = null + while (!stop){ + loop++ + def doc = Jsoup.parse(driver.getPageSource()); + lis = JQueryUtils.select(className, doc) + if(lis || loop>=10){ + stop = true; + }else{ + Thread.sleep(5000) + } + } + return lis + + } + + def parseDetail(driver){ + def trs = checkElement(driver, ".rooms tr"); + def rows = [] + for(def tr : trs){ + def tds = tr.select("td") + if(tds){ + def floor = null + for(def j=0;j + data.put(k,v) + } + map.each{k,v-> + data.put(k,v) + } + data.put("c_date", new Date()) + datas << data + } + } + if(datas){ + loudongConn.insert(datas) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy new file mode 100644 index 000000000..8a9e81f31 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookCrawl.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.thirdPriceCrawl.UserAgents +import com.fangjia.script.util.ABuYunUtil +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.jsoup.Jsoup +import org.jsoup.select.Elements + +class HouseBookCrawl { + + public def CHARSET = "utf-8"; + + static main(args) { + def task = new HouseBookCrawl() + task.run() + } + +// def run(){ +// goTocitypage() +// } +// def goTocitypage(){ +// def url = "http://www.house-book.com.cn/mainAction/mainAction/city/%25E9%2587%258D%25E5%25BA%2586%25E5%25B8%2582.html" +// def headers = [:] +// headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"); +// headers.put("User-Agent", UserAgents.get()) +// headers.put("cookCity", "%2525E9%252587%25258D%2525E5%2525BA%252586%2525E5%2525B8%252582") +// headers.put("HBJSESSIONID","afb32a63-796e-4bbc-bf39-0f8cb188079e") +// def params = [:] +// def responseMap = ABuYunUtil.doPostResponse(url, null, headers, CHARSET, false) +// def cookie = getCookie(responseMap) +// } +// +// def getCookie(responseMap) { +// def cookieAuth = null +// try { +// def response = responseMap.get("response"); +// if (StringUtils.isNotEmpty(response)) { +// JSONObject reJson = JSONObject.fromObject(response) +// //登录成功获取COOKIE +// if (reJson.success) { +// responseMap.get("headerFields").each { k, v -> +// if (k && k.equals("Set-Cookie")) { +// if (v) { +// cookieAuth = v.join(';') +// } +// } +// } +// } +// } +// } catch (Exception e) { +// println "goApiLogin(${responseMap})发生异常, 异常信息为:" + e.getMessage() +// } +// println("最终构造cookieAuth为:" + cookieAuth) +// return cookieAuth +// } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy new file mode 100644 index 000000000..1e7978919 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/ImageTest.groovy @@ -0,0 +1,147 @@ +package com.fangjia.script.loudong + +import net.sourceforge.tess4j.ITesseract +import net.sourceforge.tess4j.Tesseract +import net.sourceforge.tess4j.util.ImageHelper +import org.apache.commons.io.FileUtils +import org.ehcache.xml.model.TimeUnit +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.OutputType +import org.openqa.selenium.Point +import org.openqa.selenium.TakesScreenshot +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver + +import javax.imageio.ImageIO +import java.awt.image.BufferedImage + +public class ImageTest { + + public static void main(String[] args) throws IOException { +// File imageFile = new File("D:\\gaojuan\\img\\WaterMark.gif"); +// Tesseract instance = new Tesseract(); +// + +// //将验证码图片的内容识别为字符串 +// String result = instance.doOCR(imageFile); +// println result + + //1.获取网页的浏览器driver + //(1)定义url + String url="http://cqfdc.cq315house.com/HtmlPage/ShowRooms.html?buildingid=2l/xVhsMJPAZKivgt8ciOA&block=3%E5%8F%B7%E6%A5%BC#"; + //(2)设置driver驱动chromedriver.exe(版本:2.39)路径(相对路径) + String path = System.getProperty("user.dir"); + System.setProperty("webdriver.chrome.driver","C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + //(3)建立selenium 驱动 + //WebDriver driver = new ChromeDriver(options); + WebDriver driver = new ChromeDriver(); + //(4)设置窗口最大化及网页等待时间 + driver.manage().window().maximize(); //窗口最大化 +// driver.manage().timeouts().implicitlyWait(10, TimeUnit.SECONDS); + //(5)启动driver浏览网页 + driver.get(url); //输入网址 + Thread.sleep(1000); + + driver.findElement(By.id("cell1-11-2")).click() + Thread.sleep(1000); + + def docment = Jsoup.parse(driver.getPageSource()); + println docment + + WebElement ele = driver.findElement(By.id("vimg")); + + File screenshot = ((TakesScreenshot)driver).getScreenshotAs(OutputType.FILE); + BufferedImage pic1 = ImageIO.read(screenshot); + Point point = ele.getLocation(); + + //(2)输入验证码 + File srcfile = ((TakesScreenshot) driver).getScreenshotAs(OutputType.FILE); + FileUtils.copyFile(srcfile, new File("picture/source.png")); + //原图片 + //String soursePicPath = "D:\\data_dev\\test\\test.png"; + String soursePicPath = "picture/source.png"; + File sourcePic = new File(soursePicPath); + int eleWid = ele.getSize().getWidth() + int eleHeiht = ele.getSize().getHeight() + + BufferedImage pic2 = pic1.getSubimage(point.getX(), point.getY(), eleWid, eleHeiht); + ImageIO.write(pic2, "jpg", sourcePic); + + //File desImage = new File("D:\\data_dev\\test\\test02.png"); + File desImage = new File("picture/answer.png"); + + + + //加载待读取图片 + //File imageFile = new File("D:\\data_dev\\test\\test02.png"); + File imageFile = new File("picture/answer.png"); + BufferedImage image = ImageIO.read(imageFile); + //对图片进行处理 + image = convertImage(image); + //创建tess对象 + ITesseract instance = new Tesseract(); + //设置训练文件目录 + //instance.setDatapath("E:\\workspace\\workspace_idea\\com_mth_data\\m-kuaidi100-data\\tessdata"); + String path02 = System.getProperty("user.dir"); + instance.setDatapath(path02 + "\\tessdata"); + //设置训练语言 + //instance.setLanguage("chi_sim"); + instance.setLanguage("eng"); + //执行转换 + String s = instance.doOCR(image); + + System.out.println(s); + String[] split = s.replace(" ", "").split("=|:"); + String a = split[split.length - 1].trim(); + char[] chars = a.toCharArray(); + if (chars.length == 2) { + if (chars[0] == '7') { + chars[0] = '1'; + } + } + a = String.valueOf(chars); + + System.out.println(a); + String b = s.substring(0, 1); + System.out.println(b); + String answer = String.valueOf(Integer.parseInt(a.trim()) - Integer.parseInt(b.trim())); + + + WebElement input2 = driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_TextBox2\"]")); + input2.sendKeys(answer); + Thread.sleep(4000); + //(3)点击"查询"按钮 + driver.findElement(By.xpath("//*[@id=\"ctl00_Content_Body_Button1\"]")).click(); + + String SecondtHandle = driver.getWindowHandle(); //首先得到最先的窗口 权柄 + for (String winHandle1 : driver.getWindowHandles()) { //得到浏览器所有窗口的权柄为Set集合,遍历 + if (!winHandle1.equals(SecondtHandle)) { //如果为 最先的窗口 权柄跳出 + driver.close(); + driver.switchTo().window(winHandle1); //如果不为 最先的窗口 权柄,将 新窗口的操作权柄 给 driver + System.out.println(driver.getCurrentUrl()); //打印是否为新窗口 + } + } + + Thread.sleep(4000); + + //4.关闭浏览器 + driver.close(); + + } + + + //对图片进行处理 - 提高识别度 + public static BufferedImage convertImage(BufferedImage image) throws Exception { + //按指定宽高创建一个图像副本 + //image = ImageHelper.getSubImage(image, 0, 0, image.getWidth(), image.getHeight()); + //图像转换成灰度的简单方法 - 黑白处理 + image = ImageHelper.convertImageToGrayscale(image); + //图像缩放 - 放大n倍图像 + image = ImageHelper.getScaledInstance(image, image.getWidth() * 15, image.getHeight() * 15); + return image; + } + + +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy new file mode 100644 index 000000000..9f532247a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy @@ -0,0 +1,183 @@ +package com.fangjia.script.temp.shenjian + +import com.alibaba.excel.EasyExcel +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject + +import java.util.concurrent.* + +/** + * 成交详情导出 + * + * @author shenjian* @since 2021/3/29 + */ +class DealPriceDetailExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017) + static final Map retMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(10) + static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30) + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州", + "合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台", + "徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁", + "南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华", + "昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城", + "芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州", + "连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口", + "西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"] +// def cities = ["常州"] + + def headArr = ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量", "小区均价", "比率"] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + + CountDownLatch latch = new CountDownLatch(cities.size()) + DBObject filter = new BasicDBObject() + filter.put("city", 1) + filter.put("region", 1) + filter.put("district_name", 1) + filter.put("total_price", 1) + filter.put("trade_date", 1) + filter.put("avg_price", 1) + filter.put("area", 1) + filter.put("direction", 1) + filter.put("fitment", 1) + filter.put("height", 1) + filter.put("floor", 1) + filter.put("unit_num", 1) + filter.put("house_num", 1) + filter.put("room_num", 1) + filter.put("hall", 1) + filter.put("room", 1) + filter.put("toilet", 1) + for (def city : cities) { + final currCity = city + executorService.execute(new Runnable() { + @Override + void run() { + try { + final def ret = new CopyOnWriteArrayList() + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", currCity).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + // 小区可能存在多个别名 + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + //ret << ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量"] + DBObject query = new BasicDBObject(); + query.put("city", currCity) + query.put("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))) + + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", currCity).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).count()) + fangjiaConn.find(query, filter).batchSize(10000).each { tempObj -> + final obj = tempObj + districtExecutorService.execute(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.region + obj.district_name)) { + def row = [] + row[0] = currCity + row[1] = obj.region + row[2] = obj.district_name + row[3] = obj.total_price + row[4] = obj.avg_price + if (obj.trade_date) { + row[5] = DateUtils.formatDateByPattern(obj.trade_date, "yyyy-MM-dd") + } + row[6] = obj.area + row[7] = obj.direction + row[8] = obj.fitment + row[9] = obj.height + row[10] = obj.floor + row[11] = obj.unit_num + row[12] = obj.house_num + row[13] = obj.room_num + row[14] = obj.hall + row[15] = obj.room + row[16] = obj.toilet + def objOur = map.get(obj.region + obj.district_name) + calcOurAvgPrice(currCity, obj.trade_date, obj.avg_price, objOur, row) + ret.add(row) + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + println("出现异常111111.。。。。。。。。。。。") + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + }) + } + districtLatch.await() + EasyExcel.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx").head(head).sheet("成交信息").doWrite(ret); + //ExcelUtils.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx", ret, "成交信息") + println("当前处理城市" + currCity + " " + ret.size()) + } catch (Exception ex) { + println("出现异常22222.。。。。。。。。。。。") + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + executorService.shutdown() + districtExecutorService.shutdown() + } + + private static calcOurAvgPrice(def city, def tradeDate, def avgPrice, def objOur, def row) { + // 最相近一周的该小区均价 + Date nextWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", objOur.region) + dbObject.put("district_name", objOur.name) + // 不可用append,会被覆盖 + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + objOur.region + " " + objOur.name) + row[17] = "" + row[18] = "" + } else { + row[17] = weekendAvgPrice + row[18] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(weekendAvgPrice) / Double.valueOf(avgPrice)) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy new file mode 100644 index 000000000..fb9b88402 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy @@ -0,0 +1,158 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCursor +import org.apache.commons.lang3.time.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class PPE15Export { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30) + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["扬州"] + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + CountDownLatch latch = new CountDownLatch(cities.size()) + for (def cityTemp : cities) { + final city = cityTemp + println("当前处理城市:" + city) + executorService.execute(new Runnable() { + @Override + void run() { + try { + AtomicInteger totalCount = new AtomicInteger(0) + AtomicInteger count = new AtomicInteger(0) + def row = [] + row[0] = city + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + // 小区可能存在多个别名 + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).count()) + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).each { obj -> + districtExecutorService.execute(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.region + obj.district_name)) { + def objOur = map.get(obj.region + obj.district_name) + Double avgPrice = obj.avg_price + Date tradeDate = obj.trade_date + // 最相近一周的该小区均价 + Date nextWeekDate = DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", obj.region) + dbObject.put("district_name", objOur.name) + // 不可用append,会被覆盖 + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + dbObject.put("region", obj.region) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + +// DBCursor dbCursor = collection.find(new BasicDBObject("city", city).append("region", objOur.region).append("district_name", objOur.name) +// .append("weekend", new BasicDBObject('$lte', nextWeekDate)) +// .append("weekend", new BasicDBObject('$gte', firstWeekDate))).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name) + } else { + boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15 + if (ppe15) { + count.incrementAndGet() + } + } + totalCount.incrementAndGet() + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + }) + }; + districtLatch.await() + row[6] = totalCount.get() + row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)" + retMap.put(city, row) + + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + }; + latch.await() + for (def city: cities) { + if (retMap.get(city)) { + result << retMap.get(city) + } else { + def row = [] + row[0] = city + row[6] = 0 + row[7] = 0 + result << row + } + } + ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + executorService.shutdown() + districtExecutorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy new file mode 100644 index 000000000..40a866be1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy @@ -0,0 +1,130 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCursor +import org.apache.commons.lang3.time.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class PPE15ExportTest { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + + static main(args) { +// def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州", +// "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", +// "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", +// "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", +// "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", +// "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", +// "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", +// "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def cities = ["常州"] + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + for (def cityTemp : cities) { + final city = cityTemp + println("当前处理城市:" + city) + try { + AtomicInteger totalCount = new AtomicInteger(0) + AtomicInteger count = new AtomicInteger(0) + def row = [] + row[0] = city + // 小区 + final def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + int ppe15Count = 0 + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24")))) + .batchSize(10000).count()) + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24")))) + .batchSize(10000).each { obj -> + try { + if (map.get(obj.region + obj.district_name)) { + def objOur = map.get(obj.region + obj.district_name) + Double avgPrice = obj.avg_price + Date tradeDate = obj.trade_date + // 最相近一周的该小区均价 + Date nextWeekDate = DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", obj.region) + dbObject.put("district_name", objOur.name) + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + dbObject.put("region", obj.region) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name) + } else { + boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15 + if (ppe15) { + count.incrementAndGet() + } + } + totalCount.incrementAndGet() + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + districtLatch.await() + row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)" + retMap.put(city, row) + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + + } + }; + for (def city: cities) { + if (retMap.get(city)) { + result << retMap.get(city) + } else { + result << 0 + } + } + ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy new file mode 100644 index 000000000..dbcf2139f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy @@ -0,0 +1,153 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 上海房屋租金导出 + * + * @author shenjian + * @since 2021/03/22 + */ +class RentExport { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static Map retMap = new ConcurrentHashMap() + + static main(args) { + def result = [] + result << ["城市","区域","小区名称","房型","租金(月)","计算方式", "经度", "纬度"] + + int count = DBUtils.seaweed.find(new BasicDBObject("city", "上海"), + new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1)) + .batchSize(10000).count() + AtomicInteger size = new AtomicInteger(count) + println("上海小区" + size.get()) + CountDownLatch latch = new CountDownLatch(count) + DBUtils.seaweed.find(new BasicDBObject("city", "上海"), + new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1)) + .batchSize(10000).each {obj -> + executorService.submit(new Runnable() { + @Override + void run() { + try { + DBObject query = new BasicDBObject() + query.append("city", "上海").append("district_name", obj.name).append("region", obj.region) + .append("end_month_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2020-09-22"))) + .append("house_type", new BasicDBObject('$lt', 5)) + + DBObject filter = new BasicDBObject() + filter.append("region", 1).append("city", 1).append("district_name", 1).append("avg_price", 1) + .append("house_type", 1).append("sample_enum", 1).append("_id", 0).append("end_month_date", 1) + + buildResult(query, filter, result, obj.lng, obj.lat) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + println("*********************" + size.decrementAndGet()) + } + } + }) + } + + latch.await() + result.addAll(retMap.values()) + com.fangjia.script.util.ExcelUtils.write("G:\\数据\\建行\\租金\\租金.xlsx", result, "租金"); + executorService.shutdown() + } + + static boolean buildResult (def query, def filter, def result, def lng, def lat) { + collection.find(query, filter).batchSize(10000).sort(new BasicDBObject("end_month_date", -1).append("house_type", 1)).each {obj -> + def row = [] + row[0] = obj.city + row[1] = obj.region + row[2] = obj.district_name + row[3] = obj.house_type + row[4] = obj.avg_price + row[5] = obj.sample_enum + row[6] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lng / 10000000000d) + row[7] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lat / 10000000000d) + + if (!retMap.containsKey(obj.district_name + obj.region + obj.house_type)) { + // 判断价格是否合理 + if (isPriceReasonable(obj.district_name, obj.region, obj.house_type, obj.avg_price)) { + retMap.put(obj.district_name + obj.region + obj.house_type, row) + println(retMap.size()) + } else { + println("当前价格不合理,被过滤掉" + obj.district_name + obj.region + obj.house_type) + } + } + } + } + + static boolean isPriceReasonable(def district_name, def region, def house_type, def avg_price) { + if (retMap.contains(district_name + region + "1")) { + printf("一居室" + retMap.get(district_name + region + "1")[4]) + } + if (retMap.contains(district_name + region + "2")) { + printf("二居室" + retMap.get(district_name + region + "2")[4]) + } + if (retMap.contains(district_name + region + "3")) { + printf("三居室" + retMap.get(district_name + region + "3")[4]) + } + println("当前居室" + house_type + ", 价格为" + avg_price) + switch (house_type) { + case 2: + println(retMap.get(district_name + region + "1")) + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + case 3: + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "2")) { + double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4]) + if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + case 4: + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "2")) { + double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4]) + if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "3")) { + double threeAvgPrice = Double.valueOf(retMap.get(district_name + region + "3")[4]) + if (threeAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + } + return true + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index 8627efc19..34df24342 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.time.DateFormatUtils /** * 抓取总量监控 @@ -37,7 +38,16 @@ class SeawaterMonitor extends BaseBatch { static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] static void main(String[] args) { - runBatch(new SeawaterMonitor(), args) + //runBatch(new SeawaterMonitor(), args) + //monitorDayCrawlReasonable() + def results = [] + results << ["日期", "网站", "出售", "出租"] + for (int i = 1; i < 90; i++) { + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i).getTime() + def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime() + countDayCrawlBySite(startDate, endDate, results) + } + ExcelUtils.write("f://数据//爬虫//分析.xlsx", results, "sheet1") } /** @@ -125,6 +135,52 @@ class SeawaterMonitor extends BaseBatch { return "" } + /** + * 统计近90天更新量超过40万的天数 + * @param args + */ + static def monitorDayCrawlReasonable() { + def map = [:] + for (int i = 1; i < 90; i++) { + def date = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i) + def startDate = date.getTime(); + def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime() + def total = countDayCrawl(startDate, endDate) + if (total >= expectNumber) { + map.put(DateUtil.formatToYYYYMMDD(date), total) + } + } + println("近90天更新数量超过" + expectNumber + "的天数为" + map.size() + ",分别为:" ) + for (def key : map.keySet()) { + println(key + " : " + map.get(key)) + } + } + + /** + * 统计每个网站时间段范围内的更新量 + */ + static def countDayCrawlBySite(def startDate, def endDate, def results) { + def sites = ["lianjia.com", "ke.com", "fang.com", "58.com", "anjuke.com"] + def sql = "select count(*) as total, root, cat from seawater where m_date >= '${startDate}' and m_date < '${endDate}' group by root, cat" + def map = [:] + ElasticUtils.executeSqlQuery(sql,{data-> + map.put(data.root + "|" + data.cat, data.total) + }); + for (def site : sites) { + results << [DateFormatUtils.format(startDate, "yyyy-MM-dd"), site, map.get(site + "|sell"), map.get(site + "|rent")] + } + return results + }; + + static def countDayCrawl(def startDate, def endDate) { + def total = 0 + def sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}'" + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + return total + }; + @Override void process(Object args) { def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy index fecb21729..e638088fe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy @@ -19,6 +19,8 @@ class SnapshotMonitor extends BaseBatch { // 运维组 运维警报 static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + // 房金部自动检测机器人 + // https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736 static void main(String[] args) { runBatch(new SnapshotMonitor(), args) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy new file mode 100644 index 000000000..517fcf4ed --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -0,0 +1,15 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + +/** + * @author shenjian* @since 2021/5/18 + */ +class Temp { + + static void main(String[] args) { + // 121.431870,31.208610 + List list = GaoDeUtils.searchPoi("上海", "浦东新区新场实验小学新城校区") + print(list) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy new file mode 100644 index 000000000..b0e7a120e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy @@ -0,0 +1,92 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class TotalTestExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["贵阳"] + + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + //CountDownLatch latch = new CountDownLatch(cities.size()) + for (def city : cities) { + try { + def row = [] + row[0] = city + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, 1) + if (obj.alias) { + for (def a : obj.alias) { + map.put(obj.region + a, 1) + } + } + } + } + + + // 测试数据量(小区近一年成交量) + int tradeCount = 0 + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).each { obj -> + if (map.get(obj.region + obj.district_name)) { + tradeCount++ + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } + row[6] = tradeCount + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + //int ppe15 = + + //row[7] = + + result << row + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + //latch.countDown() + } + } + //latch.await() + ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy new file mode 100644 index 000000000..82f000f52 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class ValueRateCityInfoExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["重庆","郑州", "长沙", "北京"] + + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + //CountDownLatch latch = new CountDownLatch(cities.size()) + for (def city : cities) { + try { + def row = [] + row[0] = city + // 小区数量 + int seaweedCount = DBUtils.seaweed.count( new BasicDBObject("city", city).append("cat", "district").append("visible", 0)) + + // 可估值小区数量 + int valuateCount = collection.find(new BasicDBObject("city", city) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + ).count() + + // 有样本估值数量及占比 + int haveSampleCount = collection.find(new BasicDBObject("city", city) + .append("value_type", 2) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + ).count() + + // 无样本估值数量及占比 + int notHaveSampleCount = valuateCount - haveSampleCount + + // 可信度大于0.5的数量 + int creditCount = collection.find(new BasicDBObject("city", city) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + .append("credit", new BasicDBObject('$gte', 0.5d)) + ).count() + + row[1] = seaweedCount + row[2] = valuateCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(valuateCount * 100) / Double.valueOf(seaweedCount)) + "%)" + if (valuateCount != 0) { + row[3] = haveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(haveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)" + row[4] = notHaveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(notHaveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)" + row[5] = creditCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(creditCount * 100) / Double.valueOf(valuateCount)) + "%)" + } + + result << row + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + //latch.countDown() + } + } + //latch.await() + ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.0-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy new file mode 100644 index 000000000..7b400357a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.address + +import com.alibaba.excel.EasyExcel +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 导出标准化输出结果 + * + * @author shenjian + * @since 2021/6/22 + */ +class ExportStandardResult { + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017) + + static void main(String[] args) { + List> data = new ArrayList<>() + def map = new ConcurrentHashMap() + // 查询条件 + DBObject query = new BasicDBObject('status', 0).append('flag', '浙商疑问地址复测') + CountDownLatch latch = new CountDownLatch(conn136.count(query).intValue()) + conn136.find(query).batchSize(10000).each { + objT -> + final obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def ret = [] + ret << obj.format_province + ret << obj.format_city + ret << obj.format_region + ret << obj.format_dev + ret << obj.format_township + ret << obj.format_community + ret << obj.format_group + ret << obj.format_street + ret << obj.format_streetNumber + ret << obj.format_name + ret << obj.format_buildingNumber + ret << obj.format_unitNumber + ret << obj.format_roomNumber + ret << obj.format_floor + ret << obj.format_address + + map.put(obj.order, ret) + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + for (int i = 0; i < map.size(); i++) { + data.add(map.get(i)) + } + def headArr = ["省","市","区","开发区","街道乡镇","居委会/社区", "组","路","路号","名称","楼栋", "单元","室号","楼层","优化后标准化地址"] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + EasyExcel.write("F:\\工作\\基础建设\\标准化\\浙商疑问地址复测.xlsx").head(head).sheet("测试数据").doWrite(data); + + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy new file mode 100644 index 000000000..7993a0b8f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy @@ -0,0 +1,74 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.ProvinceData +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils + +final DBCollection townConn = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018) + +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址(估家科技)V2.xlsx", { + List item -> + def id = item['A'] + def province = item['D'] + def city = item['E'] + def area = item['F'] + String township = item['G'] + String community = item['H'] + if (StringUtils.isBlank(city)) { + return ; + } + if (item['T'] != null) { + return ; + } +// if (community == null) { +// return ; +// } + if (township) { + if (township.endsWith("镇") || township.endsWith("乡")) { + item['T'] = 2 + } else if (township.endsWith("街道")) { + item['T'] = 1 + } + } +// community = community.replaceAll("居委会", "") +// community = community.replaceAll("居民委员会", "") +// community = community.replaceAll("村民委员会", "") +// community = community.replaceAll("村委会", "") +// +// if (ProvinceData.isGovernmentCity(city)) { +// townConn.find(new BasicDBObject("province", "重庆市") +// .append("region", area + "区") +// .append("street", township) +// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each { +// obj -> +// item['T'] = obj.juweihui_type +// } +// if (item['T'] == null) { +// townConn.find(new BasicDBObject("province", "重庆市") +// .append("region", area + "区") +// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each { +// obj -> +// item['T'] = obj.juweihui_type +// } +// } +// } else { +// townConn.find(new BasicDBObject("city", city + "市") +// .append("region", area + "区") +// .append("street", township) +// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each { +// obj -> +// item['T'] = obj.juweihui_type +// } +// if (item['T'] == null) { +// townConn.find(new BasicDBObject("city", city + "市") +// .append("region", area + "区") +// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each { +// obj -> +// item['T'] = obj.juweihui_type +// } +// } +// } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillNullTownshipAndComm.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillNullTownshipAndComm.groovy new file mode 100644 index 000000000..13e8d4b8b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillNullTownshipAndComm.groovy @@ -0,0 +1,23 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.script.util.ExcelUtils + +Map> map = new HashMap<>() +List> readList = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\(曹合1 )重庆农商行测试地址V8-1(街道、社区、街道代码补充)处理--新-芝.xlsx") +for (List result : readList) { + map.put(result.get(0), result) +} +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", { + List item -> + def id = item['A'] + def township = item['G'] + def community = item['H'] + if (org.apache.commons.lang.StringUtils.isBlank(township)) { + item['G'] = map[id]['F'] + item['Y'] = '人工乡镇' + } + if (org.apache.commons.lang.StringUtils.isBlank(community)) { + item['H'] = map[id]['G'] + item['Z'] = '人工社区' + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy new file mode 100644 index 000000000..eca8609f9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy @@ -0,0 +1,57 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang.StringUtils + +/** + * 补充详细地址中缺失的乡镇街道与居委社区 + */ +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", { + List item -> + def area = item['F'] + def township = item['G'] + def community = item['H'] + String detailAddr = item['I'] + + + if (detailAddr == '重庆市云阳县双江街道恒生场2-02') { + print("test") + } + // 不包含街道与居委 + if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + !detailAddr.contains(township) && !detailAddr.contains(community) ) { + if (StringUtils.isNotBlank(area)) { + if (detailAddr.indexOf(area + "区") > 0) { + int index = detailAddr.indexOf(area + "区") + area.length() + 1 + item['K'] = detailAddr.substring(0, index) + township + community + + detailAddr.substring(index) + } else if (detailAddr.indexOf(area + "县") > 0) { + int index = detailAddr.indexOf(area + "县") + area.length() + 1 + item['K'] = detailAddr.substring(0, index) + township + community + + detailAddr.substring(index) + } + } + // 不包含社区 + } else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + detailAddr.contains(township) && !detailAddr.contains(community)) { + if (detailAddr.indexOf(township) > 0) { + int index = detailAddr.indexOf(township) + township.length() + item['K'] = detailAddr.substring(0, index) + community + + detailAddr.substring(index) + } + } // 只不包含街道 + else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + !detailAddr.contains(township) && detailAddr.contains(community)) { + if (StringUtils.isNotBlank(area)) { + if (detailAddr.indexOf(area + "区") > 0) { + int index = detailAddr.indexOf(area + "区") + area.length() + 1 + item['K'] = detailAddr.substring(0, index) + township + + detailAddr.substring(index) + } else if (detailAddr.indexOf(area + "县") > 0) { + int index = detailAddr.indexOf(area + "县") + area.length() + 1 + item['K'] = detailAddr.substring(0, index) + township + + detailAddr.substring(index) + } + } + } +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy new file mode 100644 index 000000000..b8833c3fa --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils + +import java.util.regex.Pattern + +//ExcelUtils.removeEmptyRow("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V10(优化补充社区冗余信息).xlsx") +ners = ["houseno", "cellno", "floorno", "roomno"] +Pattern GROUP_PATTERN = Pattern.compile("[一二三四五六七八九十甲乙丙丁\\d]+组") +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx",{List item-> + + if (StringUtils.isNotBlank(item["K"])) { + item["I"] = item["K"] + } +// def addr = item["B"].toString() +// def detailAddr = item["I"]?.toString() +// def pd = item["C"] +// item['J'] = "" +// if (StringUtils.isNotBlank(pd)) { +// int len = addr.lastIndexOf(pd) +// if (len > 3) { +// String aim = addr.substring(len - 3, len) +// int aimIndex = detailAddr.lastIndexOf(aim) +// if (aimIndex > 0) { +// item['J'] = detailAddr.substring(0, aimIndex + 3) + pd +// } +// } +// } + +// if(detailAddr{ +// String group = DataUtil.match(pd, GROUP_PATTERN) +// if(group == null ){ +// return +// } +// def arabic_group = null +// if(!detailAddr.contains(group)){ +// arabic_group = CNNMFilter.qCNNumericToArabic(group) +// } +// if(group){ +// def flg = addr.substring(addr.indexOf(group) - 3, addr.indexOf(group)) +// flg = flg.replace("-", "") +// if(arabic_group != null){ +// group = arabic_group +// } +// if(!detailAddr.contains(flg + group)){ +// println(item) +// detailAddr = detailAddr.replace(group, '') +// def idx = detailAddr.lastIndexOf(flg) +// try{ +// detailAddr = detailAddr.substring(0, idx) + flg + group + detailAddr.substring(idx + flg.length()) +// item["j"] = detailAddr +// }catch(Exception e){ +// item["W"] ="组替换失败" +// } +// +// +// } +// } +// } + +// HttpRequest request = new HttpRequest() +// JSONObject result = request.getJsonContent("http://192.168.0.19:31080/FangJiaNlpService/ner",["address":addr], "utf-8") +// def index = Integer.MAX_VALUE +// for(JSONObject entity : result.entities){ +// if(entity.type in ners){ +// if(entity.start < index){ +// index = entity.start +// } +// } +// } +// if(index != Integer.MAX_VALUE){ +// item["C"] = addr.substring(index) +// println(item) +// } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy new file mode 100644 index 000000000..8611995d2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy @@ -0,0 +1,99 @@ +package com.fangjia.script.temp.shenjian.address + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 地址标准化测试 + * + * @author shenjian + * @since 2021/6/22 + */ +class StandardTest { + + static ExecutorService executorService = Executors.newFixedThreadPool(10) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017) + static final String IP = "192.168.0.19:39080" + static void main(String[] args) { + AtomicLong count = new AtomicLong(0) + // 查询条件 + DBObject query = new BasicDBObject('status', 1).append('flag', '浙商疑问地址复测') + CountDownLatch latch = new CountDownLatch(conn136.count(query).intValue()) + conn136.find(query).batchSize(3000).each {objT -> + final def obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def address = obj.address + def city = obj.city == null ? "" : obj.city + HttpRequest request = new HttpRequest() + + String result = request.get("http://" + IP + "/FangJiaAddressService/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") + JSONObject object = JSONObject.parse(result); + if (object.get("status") == 0) { + JSONObject jsonResult = (JSONObject) object.get("result"); + def format_province = jsonResult.get("province") + def format_city = jsonResult.get("city") + def format_region = jsonResult.get("district") + def format_dev = jsonResult.get("dev") + def format_township = jsonResult.get("township") + def format_community = jsonResult.get("community") + def format_group = jsonResult.get("group") + def format_street = jsonResult.get("street") + def format_subStreet = jsonResult.get("subStreet") + def format_streetNumber = jsonResult.get("streetNumber") + def format_name = jsonResult.get("name") + def format_buildingNumber = jsonResult.get("buildingNumber") + def format_unitNumber = jsonResult.get("unitNumber") + def format_floor = jsonResult.get("floor") + def format_roomNumber = jsonResult.get("roomNumber") + def format_address = jsonResult.get("formatAddress") + + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0) + .append("format_province", format_province) + .append("format_city", format_city) + .append("format_region", format_region) + .append("format_dev", format_dev) + .append("format_township", format_township) + .append("format_community", format_community) + .append("format_group", format_group) + .append("format_street", format_street) + .append("format_subStreet", format_subStreet) + .append("format_streetNumber", format_streetNumber) + .append("format_name", format_name) + .append("format_buildingNumber", format_buildingNumber) + .append("format_unitNumber", format_unitNumber) + .append("format_floor", format_floor) + .append("format_roomNumber", format_roomNumber) + .append("format_address", format_address))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "标准化输出异常"))) + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + count.incrementAndGet() + if (count.intValue() % 1000 == 0) { + Thread.sleep(5000) + println(count) + } + } + } + }) + }; + latch.await(); + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/batch/RemoveOldIndexOfEs86Batch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/batch/RemoveOldIndexOfEs86Batch.groovy new file mode 100644 index 000000000..076b33977 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/batch/RemoveOldIndexOfEs86Batch.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian.batch + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import org.apache.http.client.utils.DateUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * 只保留86机器最近一个月索引 + */ +@BatchOwner(Watcher.ShenJian) +class RemoveOldIndexOfEs86 extends BaseBatch { + + static void main(String[] args) { + runBatch(new RemoveOldIndexOfEs86(), args) + } + + @Override + void process(Object args) { + String url = "http://192.168.0.86:9200/_cat/indices" + + // 必须分开写,不能有空格 + String[] cmds = ["curl", "-XGET", url, "--user", "elastic:goojia123456"] + List> indices = execCurl(cmds) + Pattern yearMonthPattern = Pattern.compile("\\d{6}") + Pattern yearMonthPattern1 = Pattern.compile("\\d{4}\\.\\d{2}") + List indexNeedDel = new ArrayList<>() + for (List index : indices) { + String name = index.get(2) + // 忽略系统索引 + if (name.startsWith(".")) { + continue + } + // 匹配含有年月的索引 + Matcher matcher = yearMonthPattern.matcher(name) + if (matcher.find()) { + String date = matcher.group() + if (dateBeforeCurrMonth(date)) { + indexNeedDel.add(name) + } + continue + } + matcher = yearMonthPattern1.matcher(name) + if (matcher.find()) { + String date = matcher.group() + if (dateBeforeCurrMonth(date)) { + indexNeedDel.add(name) + } + } + } + for (String index : indexNeedDel) { + println "删除索引" + index + String urlPrefix = "http://192.168.0.86:9200/" + cmds = ["curl", "-XDELETE", urlPrefix + index, "--user", "elastic:goojia123456"] + execCurl(cmds) + } + } + + static List> execCurl(String[] cmds) { + ProcessBuilder process = new ProcessBuilder(cmds) + Process p + try { + List result = new ArrayList<>() + p = process.start() + BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())) + String line = null + while ((line = reader.readLine()) != null) { + result.add(line.split("{1} +")) + } + return result + } catch (IOException e) { + e.printStackTrace() + } + return null + } + + def boolean dateBeforeCurrMonth(String date) { + String dateCurr = DateUtils.formatDate(new Date(), "yyyyMM"); + if (date.contains(dateCurr)) { + return false + } + dateCurr = DateUtils.formatDate(new Date(), "yyyy.MM"); + if (date.contains(dateCurr)) { + return false + } + return true + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy new file mode 100644 index 000000000..9e3d9b54e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy @@ -0,0 +1,53 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject + +/** + * 获取商圈地标 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingAddr { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + + static void main(String[] args) { + def cities = ["苏州", "西安", "合肥"] + for (def city : cities) { + DBObject query = new BasicDBObject() + query.put("city", city) + collection.find(query).batchSize(300).each {obj -> + def id = obj._id + if (obj.lng == null || obj.lat == null) { + // 查询坐标位置,并更新 + def location = this.parserByGaode(city, obj.region + obj.name + obj.address) + if (location != null) { + DBObject updateField = new BasicDBObject() + updateField.put("lng", location.lng) + updateField.put("lat", location.lat) + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', updateField), false, true) + } + } + } + } + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy new file mode 100644 index 000000000..3f03c1c01 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy @@ -0,0 +1,98 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * 商圈地标导出 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingExport { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static void main(String[] args) { + def cityCodes = ["320500","610100","340100"]; + def cities = ["苏州", "西安", "合肥"] + def headers = ["城市", "区域", "商圈名称", "地标名称(3-5个)", "地标地址(与地标名称对应)", "地标经度(与地标名称对应)", "地标纬度(与地标名称对应)", "备注", "创建时间", "更新时间", "来源公司"]; + for (int i = 0; i < cities.size(); i++) { + final def city = cities[i]; + def result = [] + // 获取所有商圈信息 + def businessNames = collection.distinct("name", new BasicDBObject("city", city)) + // 获取商圈下3个地标 + for (def name : businessNames) { + List landmarks = new ArrayList<>() + List addresses = new ArrayList<>() + List lngs = new ArrayList<>() + List lats = new ArrayList<>() + String region = "" + String c_date + String m_date + int landmarkLen = 0; + collection.find(new BasicDBObject("city", city).append("name", name)).sort(new BasicDBObject("order", 1)).each { obj -> + // 按人气取经纬度不为空的地标 + if (landmarks.size() < 5) { + // 特殊标题去重 + if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1 + || String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1 + || String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1 + || String.valueOf(obj.landmark).indexOf(",") != -1 || String.valueOf(obj.landmark).indexOf("商圈附近") != -1 || String.valueOf(obj.landmark).indexOf("】") != -1) { + + } else { + // 防止导入导致字段过长,一个字符2个字节 + def landmark = String.valueOf(obj.landmark).replaceAll(" ", "") + if (landmark.length() <= 30 && landmarkLen + String.valueOf(landmark).length() <= 100) { + landmarkLen += String.valueOf(landmark).length() + landmarks.add(landmark) + addresses.add(obj.address) + lngs.add(obj.lng) + lats.add(obj.lat) + + if (StringUtils.isBlank(region)) { + region = obj.region + c_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss"); + m_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.m_date, "yyyy-MM-dd HH:mm:ss"); + } + } + } + } + } + def ret = [] + def city_code + def district_code + if (StringUtils.isNotBlank(region)) { + cityCollection.find(new BasicDBObject("fangjia_city", city).append("district", new BasicDBObject('$regex', region + "+")), + new BasicDBObject("city_code", 1).append("district_code", 1)).each {obj -> + city_code = obj.city_code + district_code = obj.district_code + } + } + ret[0] = city_code + ret[1] = district_code + ret[2] = name + ret[3] = StringUtils.join(landmarks, ",") + ret[4] = StringUtils.join(addresses, ",") + ret[5] = StringUtils.join(lngs, ",") + ret[6] = StringUtils.join(lats, ",") + ret[7] = "" + ret[8] = c_date + ret[9] = m_date + ret[10] = "fjw" + if (district_code != null) { + result << ret + } else { + println("未找到商圈地标:" + name) + } + } + FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy new file mode 100644 index 000000000..df968e183 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy @@ -0,0 +1,51 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * 商圈数据修正 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingModify { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + + static void main(String[] args) { + def cityCodes = ["320500","610100","340100"]; + def city = "合肥" + def origin = "政务区" + def now = "蜀山区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + + city = "苏州" + origin = "工业园区" + now = "吴中区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + + city = "苏州" + origin = "甪直" + now = "吴中区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + + city = "西安" + origin = "高新区" + now = "雁塔区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy new file mode 100644 index 000000000..1c8d2f70f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy @@ -0,0 +1,71 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils; + +/** + * + * 新增公司 + * + * @author shenjian + * @since 2021-03-16 + */ +class AddCompany { + + static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + + static final Set developCompanySet = new HashSet() + static final Set propertyCompanySet = new HashSet() + + static final Set actualDevelopCompanySet = new HashSet() + static final Set actualPropertyCompanySet = new HashSet() + + static { + develop_company.find().batchSize(10000).each { obj -> + developCompanySet.add(obj.company) + } + property_company.find().batchSize(10000).each { obj -> + propertyCompanySet.add(obj.company) + } + } + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0).append("city", "哈尔滨") + DbUtils.seaweed.find(query, + new BasicDBObject("developer", 1).append("estate_company", 1)) + .batchSize(10000).each {obj -> + actualDevelopCompanySet.add(obj.developer) + actualPropertyCompanySet.add(obj.estate_company) + } + + def developCompanySaves = [] + def propertyCompanySaves = [] + for (String name : actualDevelopCompanySet) { + if (StringUtils.isBlank(name) || developCompanySet.contains(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + developCompanySaves << item + } + for (String name : actualPropertyCompanySet) { + if (StringUtils.isBlank(name) || propertyCompanySet.contains(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + propertyCompanySaves << item + } + + + develop_company.insert(developCompanySaves) + property_company.insert(propertyCompanySaves) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy new file mode 100644 index 000000000..fb1e94c2b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy @@ -0,0 +1,33 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils; + +/** + * + * 开发商公司 + * + * @author shenjian + * @since 2021-03-13 + */ +class DevelopCompany { + + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136") + static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + + static void main(String[] args) { + def companySaves = [] + List result = seaweed.distinct("developer") + for (String name : result) { + if (StringUtils.isBlank(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + companySaves << item + } + develop_company.insert(companySaves) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy new file mode 100644 index 000000000..dca212b14 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy @@ -0,0 +1,34 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils + +/** + * 物业公司 + * + * @author shenjian + * @since 2021-03-13 + */ +class PropertyCompany { + + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136") + static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + + static void main(String[] args) { + // 物业公司 + def companySaves = [] + List result = seaweed.distinct("estate_company") + for (String name : result) { + if (StringUtils.isBlank(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + companySaves << item + } + property_company.insert(companySaves) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy new file mode 100644 index 000000000..0ed85bb0a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy @@ -0,0 +1,74 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 重庆所有小区周边导出 + * + * @author shenjian + * @since 2021/5/15 + */ +class AllExport { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = 3000 + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.put("city", "重庆") + query.put("cat", "district") + query.put("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + AtomicInteger count = new AtomicInteger(0) + DBUtils.seaweed.find(query, filter).batchSize(10000).each {district -> + executorService.execute(new Runnable() { + @Override + void run() { + def city = "重庆" + def name = district.name + def region = district.region + boolean exist = conn.count(new BasicDBObject("matched_city": city, "matched_region": region,"matched_name": name)) > 0 ? true : false + if (!exist) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.lng / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.lat / 10000000000d) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("matched_city", city) + obj.put("matched_region", district.region) + obj.put("matched_name", district.name) + } + conn.insert(arounds) + } + println("当前处理小区" + name + " " + count.incrementAndGet()) + } else { + println("小区已存在") + } + } + }) + } + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy new file mode 100644 index 000000000..28cec6534 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -0,0 +1,81 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import cn.emay.mina.util.ConcurrentHashSet +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区周围信息导出 + * + * @author shenjian + * @since 2021/5/15 + */ +class AroundExport { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = 3000 + //static ExecutorService executorService = Executors.newFixedThreadPool(30) + + // 高德地图中不存在周边的数据 + // 重庆 江北 阳光丽苑 + //重庆 江北 金田花苑 + //重庆 巴南 巴滨一号璀璨滨江 + //重庆 丰都 茅林沟移民新村 + //重庆 江津 福城北苑 + //重庆 巫山 摩天岭壹号 + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 0) + AtomicInteger count = new AtomicInteger(0) + for (List districtT : data) { + final List district = districtT +// executorService.execute(new Runnable() { +// @Override +// void run() { + def city = district.get(0) + def region = district.get(1) + def name = district.get(2) + boolean exist = conn.count(new BasicDBObject("matched_city": city, "matched_region": region,"matched_name": name)) > 0 ? true : false + if (!exist) { + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("region", region) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", name) + DBUtils.seaweed.find(query, new BasicDBObject("lat", 1).append("lng", 1)).limit(1).each {objT -> + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, objT.lng / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, objT.lat / 10000000000d) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("matched_city", city) + obj.put("matched_region", region) + obj.put("matched_name", name) + } + conn.insert(arounds) + } + println("当前处理小区" + name + " " + count.incrementAndGet()) + } + } else { + println("小区已存在") + } +// } +// }) + } + //executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy new file mode 100644 index 000000000..3bffc1300 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -0,0 +1,189 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.alibaba.excel.EasyExcel +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 周边数据导出新方式 + * + * @author shenjian + * @since 2021/5/16 + */ +class AroundExportByNewWay { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + AtomicInteger count = new AtomicInteger(0) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def id = district.get(0) + def district_code = district.get(2) + def name = district.get(3) + List result = [] + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) + + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + Set set = new HashSet<>() + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id, set) + } + } + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) + } + resultMap.put(district_code + name, result) + println("当前处理" + count.incrementAndGet() ) + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + }; + latch.await(); + def headArr = ['IDB', '名称','类型', '距离'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + Map map = new HashedMap() + Integer key = 1; + def ret = [] + AtomicInteger totalCount = new AtomicInteger() + for (List district : data) { + + def district_code = district.get(2) + def name = district.get(3) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() + List currRet = resultMap.get(district_code + name) + Collections.sort(currRet) + ret.addAll(currRet) + if (ret.size() > 700000) { + map.put(key++, ret) + ret = [] + } + } + } + map.put(key++, ret) + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + for (Integer k : map.keySet()) { + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套(文件"+ k + ")V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + print(totalCount.get() + " " + (totalCount.get() / data.size())) + printUnMatched(unMatchedMap) + executorService.shutdown() + } + + static calcAroundInfo(def lat, def lng, def result, def id, def set) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 3000) { + if (distance == 0) { + distance = 50 + } + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) + } + } + } + } + + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] + } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy new file mode 100644 index 000000000..89b1c7b3c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy @@ -0,0 +1,211 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.alibaba.excel.EasyExcel +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 周边数据一千米配套数量统计 + * + * @author shenjian + * @since 2021/5/16 + */ +class AroundFormatExport { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + AtomicInteger count = new AtomicInteger(0) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def id = district.get(0) + def district_code = district.get(2) + def name = district.get(3) + List result = [] + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) + + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + Set set = new HashSet<>() + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id, set) + } + } + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) + } + resultMap.put(district_code + name, result) + println("当前处理" + count.incrementAndGet() ) + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + }; + latch.await(); + def headArr = ['IDB', '数量'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + Map map = new HashedMap() + Integer key = 1; + def ret = [] + AtomicInteger totalCount = new AtomicInteger() + for (List district : data) { + def id = district.get(0) + def district_code = district.get(2) + def name = district.get(3) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() + List currRet = resultMap.get(district_code + name) + int gj = 0 + int dt = 0 + int yh = 0 + int yy = 0 + int sc = 0 + int xx = 0 + for (AroundDTO dto : currRet) { + if(dto.getType() == '公交') { + gj++ + } else if(dto.getType() == '地铁') { + dt++ + } else if(dto.getType() == '银行') { + yh++ + } else if (dto.getType() == '医院') { + yy++ + } else if(dto.getType() == '商场') { + sc++ + } else if(dto.getType() == '学校') { + xx++ + } + } + ret << [id, "公交:" + gj + ";" + "地铁:" + dt + ";银行:" + yh + ";医院:" + yy + ";商场:" + sc + ";学校:" + xx] + if (ret.size() > 700000) { + map.put(key++, ret) + ret = [] + } + } else { + ret << [id, ""] + } + } + map.put(key++, ret) + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + for (Integer k : map.keySet()) { + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + print(totalCount.get() + " " + (totalCount.get() / data.size())) + printUnMatched(unMatchedMap) + executorService.shutdown() + } + + static calcAroundInfo(def lat, def lng, def result, def id, def set) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 1000) { + if (distance == 0) { + distance = 50 + } + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) + } + } + } + } + + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] + } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy new file mode 100644 index 000000000..a5338340b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicInteger + +/** + * 通过匹配出的小区核对经纬度 + * + * @author shenjian + * @since 2021/5/19 + */ +class CheckLatLngByDistrict { + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + for (List districtT : data) { + final List district = districtT + if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + if (district.get(20) != null && district.get(21) != null) { + def originLng = district.get(20) + def originLat = district.get(21) + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + if (originLng != obj.lng || originLat != obj.lat) { + def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d); + def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d); + originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d); + originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d); + + double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng) + println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" ) + } + } else { + println(district.get(24) + " 不存在经纬度") + } + } + } else { + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + + } else { + println(district.get(24) + " 不存在经纬度222222222") + } + } + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy new file mode 100644 index 000000000..bfafad05d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy @@ -0,0 +1,10 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +/** + * @author shenjian + * @since 2021/5/15 + */ +class Constants { + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy new file mode 100644 index 000000000..48be458a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy @@ -0,0 +1,106 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import com.mongodb.QueryOperators + +/** + * 处理未匹配的小区 + * + * @author shenjian + * @since 2021/5/17 + */ +class DealUnMatched { + + static void main(String[] args) { + // getLatLngByName() + // getLatLngByNumber() + //findExceptionDistance() + + // B114862 + def lng = 107.736591 + def lat = 29.872502 + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 500) + DBObject q = new BasicDBObject() + q.put('city', '重庆') + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + DBUtils.seaweed.find(q).batchSize(100).each {obj -> + println(obj) + } + } + + static void findExceptionDistance() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + for (List district : data) { + if (district.size() > 12) { + def lng = java.lang.Double.valueOf(district.get(8)) + def lat = java.lang.Double.valueOf(district.get(9)) + def lng1 = java.lang.Double.valueOf(district.get(11)) + def lat1 = java.lang.Double.valueOf(district.get(12)) + double distance = GaoDeUtils.getDistance(lat, lng, lat1, lng1) + if (distance > 500) { + println("距离异常" + district) + district.add(distance) + district.add("距离异常") + } + } + } + data.add(0, new ArrayList()) + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "周边配套") + } + + static void getLatLngByNumber() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",1) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + + //def region = SeaweedUtils.getRegionByCode(district_code) + if (district.size() <= 5) { + if (number != null && number.size() > 5) { + def map = GaoDeUtils.parserByGaode(city, number) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过楼栋得出") + } + } else { + println(district_code + " " + name + " " + number) + } + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配1") + } + + static void getLatLngByName() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", 0) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + //def region = SeaweedUtils.getRegionByCode(district_code) + + def map = GaoDeUtils.parserByGaode(city, name) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过小区得出") + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy new file mode 100644 index 000000000..b8f0452ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy @@ -0,0 +1,53 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 导入中学小学信息 + * + * @author shenjian + * @since 2021/5/17 + */ +class ImportSchool { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\重庆学校_0902.xlsx", 1) + List result = new ArrayList(); + for (List school : data) { + try { + def city = school.get(0) + def region = school.get(1) + def name = school.get(2) + def address = school.get(4) + def type = school.get(6) + def lng = school.get(9) + def lat = school.get(10) + + DBObject dbObject = new BasicDBObject() + dbObject.put("province", "重庆") + dbObject.put("city", city) + dbObject.put("region", region) + dbObject.put("name", name) + dbObject.put("address", address) + dbObject.put("type", type) + dbObject.put("typecode", "141200") + dbObject.put("lng", lng) + dbObject.put("lat", lat) + dbObject.put("status", 0) + dbObject.put("flag", "excel导入") + result.add(dbObject) + } catch(Exception ex) { + println(school) + ex.printStackTrace() + } + } + + poiConn.insert(result) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy new file mode 100644 index 000000000..9ce164947 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import sun.swing.BakedArrayList + +/** + * 标记不需要导出的数据 + * + * @author shenjian + * @since 2021/5/16 + */ +class MarkUnNeedExport { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 1) + for (List districtT : data) { + final List district = districtT + if (district.size() > 1) { + def type = district.get(0) + def status = district.get(1) + if (status == 1.0) { + poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 0)), false, true) + } else { + poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 1)), false, true) + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy new file mode 100644 index 000000000..c641a4fc8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy @@ -0,0 +1,115 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import cn.emay.mina.util.ConcurrentHashSet +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * POI迁移 + * + * @author shenjian + * @since 2021/5/16 + */ +class MoveArroundTemp { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + Map map = new ConcurrentHashMap() + AtomicInteger count = new AtomicInteger() + int size = conn.count(new BasicDBObject("status", 1)) + println(size) + CountDownLatch latch = new CountDownLatch(size) + conn.find(new BasicDBObject("status", 1)).batchSize(10000).each { objT -> + final def obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.id) == null && conn43.count(new BasicDBObject("_id", obj.id)) <= 0) { + DBObject dbObject = new BasicDBObject() + dbObject.put("_id", obj.id) + dbObject.put("name", obj.name) + dbObject.put("type", obj.type) + dbObject.put("typecode", obj.typecode) + dbObject.put("address", obj.address) + dbObject.put("lng", Double.valueOf(obj.location.split(",")[0])) + dbObject.put("lat", Double.valueOf(obj.location.split(",")[1])) + dbObject.put("tel", obj.tel) + dbObject.put("province", obj.pname) + dbObject.put("city", obj.cityname) + dbObject.put("region", obj.adname) + dbObject.put("status", 2) + dbObject.put("flag", "待清洗") + map.put(obj.id, dbObject) + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + println(count.incrementAndGet()) + } + } + }) + } + latch.await() + List ret = new ArrayList() + for (String key : map.keySet()) { + ret << map.get(key) + } + conn43.insert(ret) + executorService.shutdown() + } + + static origin() { +// List list = Collections.synchronizedList(new ArrayList()) +// List ids = conn.distinct("id") +// CountDownLatch latch = new CountDownLatch(ids.size()) +// for (String idT : ids) { +// final String id = idT +// executorService.submit(new Runnable() { +// @Override +// void run() { +// try { +// DBObject dbObject = new BasicDBObject() +// conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> +// dbObject.put("_id", id) +// dbObject.put("name", obj.name) +// dbObject.put("type", obj.type) +// dbObject.put("typecode", obj.typecode) +// dbObject.put("address", obj.address) +// dbObject.put("lng", Double.valueOf(obj.location.split(",")[0])) +// dbObject.put("lat", Double.valueOf(obj.location.split(",")[1])) +// dbObject.put("tel", obj.tel) +// dbObject.put("province", obj.pname) +// dbObject.put("city", obj.cityname) +// dbObject.put("region", obj.adname) +// } +// list.add(dbObject) +// } catch(Exception ex) { +// ex.printStackTrace() +// } finally { +// latch.countDown() +// println("当前记录数" + list.size()) +// } +// } +// }) +// }; +// latch.await(); +// conn43.insert(list); +// list.clear(); +// println("同步清理完成"); +// executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy new file mode 100644 index 000000000..1c907e236 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.regex.Pattern + +/** + * POI清理规则 + * + * @author shenjian + * @since 2021/5/16 + */ +class POIRule { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + + } + + static void changeSchool() { + // 小学校转为学校 + poiConn.find(new BasicDBObject("typecode", "141203").append("status", 0)).batchSize(1000).each {obj -> + if (obj.name.indexOf('小学校') != -1 ) { + def id = obj._id + def region = obj.region + def name = obj.name.replace("小学校", '小学') + if (poiConn.count(new BasicDBObject("region": region, "name": name)) > 0) { + poiConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("status", 1)), false, true) + } else { + poiConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("name", name)), false, true) + } + } + } + } + + static BasicDBObject getLikeStr(String findStr){ + Pattern pattern = Pattern.compile('^.*' + findStr + '.*$', Pattern.CASE_INSENSITIVE); + return new BasicDBObject('$regex', pattern); + } + +// 更新规则 +// db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCDEFHG]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /购物服务;商场;免税品店/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;综合医院;卫生院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;学校/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[1234]期/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /住院部/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /\//, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /专科医院/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /诊所/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy new file mode 100644 index 000000000..40c42f36a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author shenjian + * @since 2021/5/17 + */ +class SavePoi { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = 3000 + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + AtomicInteger count = new AtomicInteger() + for (List listT : data) { + final list = listT + executorService.execute(new Runnable() { + @Override + void run() { + if (list.size() >= 17) { + def city = list.get(1) + def lng = Double.valueOf(list.get(11)) + def lat = Double.valueOf(list.get(12)) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("status", 1) + obj.put("flag", "距离异常入库") + } + conn.insert(arounds) + println(count.incrementAndGet()) + } + } + } + }) + }; + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy new file mode 100644 index 000000000..9817c00cb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -0,0 +1,113 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 重庆农商行 + * 开发商物业公司等小区信息导出 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeaweedExport { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.43", 28017) + + static def map = new ConcurrentHashMap() + + static void main(String[] args) { + def resultMap = new ConcurrentHashMap() + List> data = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\重庆样本匹配结果V4.xlsx") + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def no = district.get(0) + def id = district.get(2) + def ret = [] + if (id != null) { + DBObject query = new BasicDBObject() + query.put("_id", new ObjectId(id)) + //query.put("region", matchedRegion) + query.put("cat", "district") + query.put("visible", 0) + //query.put("name", matchedName) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("developer", 1) + filter.put("estate_company", 1) + filter.put("cubage_percent", 1) // 容积率 + filter.put("virescence_percent", 1) // 绿化率 + filter.put("alias", 1) + filter.put("car_site_count", 1) + filter.put("all_area", 1) + filter.put("loudong_count", 1) + filter.put("household_count", 1) + filter.put("estate_charge", 1) + filter.put("open_date", 1) // 开盘时间 + filter.put("live_date", 1) // 交房时间 + filter.put("elevator", 1) // 电梯 + + DBObject obj = seaweed.findOne(new ObjectId(id)) + if (obj) { + ret[0] = obj.estate_company + ret[1] = obj.developer + ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) + ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) + ret[4] = SeaweedUtils.formatAlias(obj.mapping_alias) + ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) + ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) + ret[7] = "" + ret[8] = SeaweedUtils.formatAllArea(obj.all_area) + ret[9] = obj.loudong_count + ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count) + ret[11] = "" + ret[12] = obj.estate_charge + ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date) + ret[14] = obj.live_date + ret[15] = obj.is_elevator == true ? "有" : (obj.is_elevator == false ? "无" : "") + ret[16] = no + resultMap.put(no, ret) + } + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + }; + latch.await(); + def ret = []; + ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "是否有电梯", "序号"]; + for (List district : data) { + def no = district.get(0) + ret << resultMap.get(no) + }; + com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\POC测试\\开发商及物业信息.xlsx", ret, "开发商及物业信息"); + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy new file mode 100644 index 000000000..5222957f7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy @@ -0,0 +1,283 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 58城市简写 + * + * @author shenjian + * @since 2021/5/10 + */ +class City58 { + + static String cityList = "{\n" + + " \"安徽\":{\n" + + " \"合肥\":\"hf|837\",\"芜湖\":\"wuhu|2045\",\"蚌埠\":\"bengbu|3470\",\"阜阳\":\"fy|2325\",\"淮南\":\"hn|2319\",\n" + + " \"安庆\":\"anqing|3251\",\"宿州\":\"suzhou|3359\",\"六安\":\"la|2328\",\"淮北\":\"huaibei|9357\",\"滁州\":\"chuzhou|10266\",\n" + + " \"马鞍山\":\"mas|2039\",\"铜陵\":\"tongling|10285\",\"宣城\":\"xuancheng|5633\",\"亳州\":\"bozhou|2329\",\"黄山\":\"huangshan|2323\",\n" + + " \"池州\":\"chizhou|10260\",\"巢湖\":\"ch|10229\",\"和县\":\"hexian|10892\",\"霍邱\":\"hq|11226\",\"桐城\":\"tongcheng|11296\",\n" + + " \"宁国\":\"ningguo|5645\",\"天长\":\"tianchang|10273\",\"东至\":\"dongzhi|10262\",\"无为\":\"wuweixian|10232\"\n" + + " }\n" + + " ,\n" + + " \"福建\":{\n" + + " \"福州\":\"fz|304\",\"厦门\":\"xm|606\",\"泉州\":\"qz|291\",\"莆田\":\"pt|2429\",\"漳州\":\"zhangzhou|710\",\n" + + " \"宁德\":\"nd|7951\",\"三明\":\"sm|2048\",\"南平\":\"np|10291\",\"龙岩\":\"ly|6752\",\"武夷山\":\"wuyishan|10761\",\n" + + " \"石狮\":\"shishi|296\",\"晋江\":\"jinjiangshi|297\",\"南安\":\"nananshi|293\",\"龙海\":\"longhai|713\",\"上杭\":\"shanghangxian|6757\",\n" + + " \"福安\":\"fuanshi|7969\",\"福鼎\":\"fudingshi|7970\",\"安溪\":\"anxixian|7100\",\"永春\":\"yongchunxian|7101\",\"永安\":\"yongan|2133\",\n" + + " \"漳浦\":\"zhangpu|717\"\n" + + " }\n" + + " ,\n" + + " \"广东\":{\n" + + " \"深圳\":\"sz|4\",\"广州\":\"gz|3\",\"东莞\":\"dg|413\",\"佛山\":\"fs|222\",\"中山\":\"zs|771\",\n" + + " \"珠海\":\"zh|910\",\"惠州\":\"huizhou|722\",\"江门\":\"jm|629\",\"汕头\":\"st|783\",\"湛江\":\"zhanjiang|791\",\n" + + " \"肇庆\":\"zq|901\",\"茂名\":\"mm|679\",\"揭阳\":\"jy|927\",\"梅州\":\"mz|9389\",\"清远\":\"qingyuan|7303\",\n" + + " \"阳江\":\"yj|2284\",\"韶关\":\"sg|2192\",\"河源\":\"heyuan|10467\",\"云浮\":\"yf|10485\",\"汕尾\":\"sw|9449\",\n" + + " \"潮州\":\"chaozhou|10461\",\"台山\":\"taishan|11263\",\"阳春\":\"yangchun|8566\",\"顺德\":\"sd|8716\",\"惠东\":\"huidong|725\",\n" + + " \"博罗\":\"boluo|726\",\"海丰\":\"haifengxian|9444\",\"开平\":\"kaipingshi|634\",\"陆丰\":\"lufengshi|9456\"\n" + + " }\n" + + " ,\n" + + " \"广西\":{\n" + + " \"南宁\":\"nn|845\",\"柳州\":\"liuzhou|7133\",\"桂林\":\"gl|1039\",\"玉林\":\"yulin|2337\",\"梧州\":\"wuzhou|2046\",\n" + + " \"北海\":\"bh|10536\",\"贵港\":\"gg|6770\",\"钦州\":\"qinzhou|2335\",\"百色\":\"baise|10513\",\"河池\":\"hc|2340\",\n" + + " \"来宾\":\"lb|10552\",\"贺州\":\"hezhou|10549\",\"防城港\":\"fcg|10539\",\"崇左\":\"chongzuo|10524\",\"桂平\":\"guipingqu|6774\",\n" + + " \"北流\":\"beiliushi|9168\",\"博白\":\"bobaixian|9173\",\"岑溪\":\"cenxi|2119\"\n" + + " }\n" + + " ,\n" + + " \"贵州\":{\n" + + " \"贵阳\":\"gy|2015\",\"遵义\":\"zunyi|7620\",\"黔东南\":\"qdn|9363\",\"黔南\":\"qn|10492\",\"六盘水\":\"lps|10506\",\n" + + " \"毕节\":\"bijie|10564\",\"铜仁\":\"tr|10417\",\"安顺\":\"anshun|7468\",\"黔西南\":\"qxn|10434\",\"仁怀\":\"renhuaishi|7628\",\n" + + " \"清镇\":\"qingzhen|12703\"\n" + + " }\n" + + " ,\n" + + " \"甘肃\":{\n" + + " \"兰州\":\"lz|952\",\"天水\":\"tianshui|8601\",\"白银\":\"by|10304\",\"庆阳\":\"qingyang|10475\",\"平凉\":\"pl|7154\",\n" + + " \"酒泉\":\"jq|10387\",\"张掖\":\"zhangye|10454\",\"武威\":\"wuwei|10448\",\"定西\":\"dx|10322\",\"金昌\":\"jinchang|7428\",\n" + + " \"陇南\":\"ln|10415\",\"临夏\":\"linxia|7112\",\"嘉峪关\":\"jyg|10362\",\"甘南\":\"gn|10343\",\"敦煌\":\"dunhuang|10390\"\n" + + " }\n" + + " ,\n" + + " \"海南\":{\n" + + " \"海口\":\"haikou|2053\",\"三亚\":\"sanya|2422\",\"五指山\":\"wzs|9952\",\"三沙\":\"sansha|13722\",\"琼海\":\"qh|10136\",\n" + + " \"文昌\":\"wenchang|9984\",\"万宁\":\"wanning|10022\",\"屯昌\":\"tunchang|10044\",\"琼中\":\"qiongzhong|10064\",\"陵水\":\"lingshui|10184\",\n" + + " \"东方\":\"df|10250\",\"定安\":\"da|10303\",\"澄迈\":\"cm|10331\",\"保亭\":\"baoting|10367\",\"白沙\":\"baish|10380\",\n" + + " \"儋州\":\"danzhou|10394\"\n" + + " }\n" + + " ,\n" + + " \"河南\":{\n" + + " \"郑州\":\"zz|342\",\"洛阳\":\"luoyang|556\",\"新乡\":\"xx|1016\",\"南阳\":\"ny|592\",\"许昌\":\"xc|977\",\n" + + " \"平顶山\":\"pds|1005\",\"安阳\":\"ay|1096\",\"焦作\":\"jiaozuo|3266\",\"商丘\":\"sq|1029\",\"开封\":\"kaifeng|2342\",\n" + + " \"濮阳\":\"puyang|2346\",\"周口\":\"zk|933\",\"信阳\":\"xy|8694\",\"驻马店\":\"zmd|1067\",\"漯河\":\"luohe|2347\",\n" + + " \"三门峡\":\"smx|9317\",\"鹤壁\":\"hb|2344\",\"济源\":\"jiyuan|9918\",\"明港\":\"mg|8541\",\"鄢陵\":\"yanling|9123\",\n" + + " \"禹州\":\"yuzhou|979\",\"长葛\":\"changge|9344\",\"灵宝\":\"lingbaoshi|9307\",\"杞县\":\"qixianqu|7389\",\"汝州\":\"ruzhou|1010\",\n" + + " \"项城\":\"xiangchengshi|935\",\"偃师\":\"yanshiqu|7121\",\"长垣\":\"changyuan|5936\",\"滑县\":\"huaxian|5405\",\"林州\":\"linzhou|1101\",\n" + + " \"沁阳\":\"qinyang|3268\",\"孟州\":\"mengzhou|3267\",\"温县\":\"wenxian|7312\",\"尉氏\":\"weishixian|7391\",\"兰考\":\"lankaoxian|7393\",\n" + + " \"通许\":\"tongxuxian|7390\",\"新安\":\"lyxinan|11217\",\"伊川\":\"yichuan|11220\",\"孟津\":\"mengjinqu|7122\",\"宜阳\":\"lyyiyang|11219\",\n" + + " \"舞钢\":\"wugang|1011\",\"永城\":\"yongcheng|1032\",\"睢县\":\"suixian|1038\",\"鹿邑\":\"luyi|939\",\"渑池\":\"yingchixian|9322\",\n" + + " \"沈丘\":\"shenqiu|942\",\"太康\":\"taikang|938\",\"商水\":\"shangshui|936\",\"淇县\":\"qixianq|9186\",\"浚县\":\"junxian|9185\",\n" + + " \"范县\":\"fanxian|7285\",\"固始\":\"gushixian|8698\",\"淮滨\":\"huaibinxian|8702\",\"邓州\":\"dengzhou|595\",\"新野\":\"xinye|603\"\n" + + " }\n" + + " ,\n" + + " \"黑龙江\":{\n" + + " \"哈尔滨\":\"hrb|202\",\"大庆\":\"dq|375\",\"齐齐哈尔\":\"qqhr|5853\",\"牡丹江\":\"mdj|3489\",\"绥化\":\"suihua|6718\",\n" + + " \"佳木斯\":\"jms|6776\",\"鸡西\":\"jixi|7289\",\"双鸭山\":\"sys|9837\",\"鹤岗\":\"hegang|9061\",\"黑河\":\"heihe|9862\",\n" + + " \"伊春\":\"yich|9773\",\"七台河\":\"qth|9848\",\"大兴安岭\":\"dxal|9878\",\"安达\":\"shanda|6720\",\"肇东\":\"shzhaodong|6721\",\n" + + " \"肇州\":\"zhaozhou|382\"\n" + + " }\n" + + " ,\n" + + " \"湖北\":{\n" + + " \"武汉\":\"wh|158\",\"宜昌\":\"yc|858\",\"襄阳\":\"xf|891\",\"荆州\":\"jingzhou|3479\",\"十堰\":\"shiyan|2032\",\n" + + " \"黄石\":\"hshi|1734\",\"孝感\":\"xiaogan|3434\",\"黄冈\":\"hg|2299\",\"恩施\":\"es|2302\",\"荆门\":\"jingmen|2296\",\n" + + " \"咸宁\":\"xianning|9617\",\"鄂州\":\"ez|9709\",\"随州\":\"suizhou|9656\",\"潜江\":\"qianjiang|9669\",\"天门\":\"tm|9517\",\n" + + " \"仙桃\":\"xiantao|9736\",\"神农架\":\"snj|9605\",\"宜都\":\"yidou|864\",\"汉川\":\"hanchuan|3439\",\"枣阳\":\"zaoyang|896\",\n" + + " \"武穴\":\"wuxueshi|7362\",\"钟祥\":\"zhongxiangshi|9119\",\"京山\":\"jingshanxian|9117\",\"沙洋\":\"shayangxian|9118\",\"松滋\":\"songzi|3484\",\n" + + " \"广水\":\"guangshuishi|9657\",\"赤壁\":\"chibishi|9623\",\"老河口\":\"laohekou|895\",\"谷城\":\"gucheng|899\",\"宜城\":\"yichengshi|897\",\n" + + " \"南漳\":\"nanzhang|898\",\"云梦\":\"yunmeng|3438\",\"安陆\":\"anlu|3442\",\"大悟\":\"dawu|3437\",\"孝昌\":\"xiaochang|3436\",\n" + + " \"当阳\":\"dangyang|865\",\"枝江\":\"zhijiang|866\",\"嘉鱼\":\"jiayuxian|9624\",\"随县\":\"suixia|9660\"\n" + + " }\n" + + " ,\n" + + " \"湖南\":{\n" + + " \"长沙\":\"cs|414\",\"株洲\":\"zhuzhou|1086\",\"益阳\":\"yiyang|10198\",\"常德\":\"changde|872\",\"衡阳\":\"hy|914\",\n" + + " \"湘潭\":\"xiangtan|2047\",\"岳阳\":\"yy|821\",\"郴州\":\"chenzhou|5695\",\"邵阳\":\"shaoyang|2303\",\"怀化\":\"hh|5756\",\n" + + " \"永州\":\"yongzhou|2307\",\"娄底\":\"ld|9481\",\"湘西\":\"xiangxi|10219\",\"张家界\":\"zjj|6788\",\"醴陵\":\"liling|1091\",\n" + + " \"澧县\":\"lixian|876\",\"桂阳\":\"czguiyang|5699\",\"资兴\":\"zixing|5698\",\"永兴\":\"yongxing|5701\",\"常宁\":\"changningshi|921\",\n" + + " \"祁东\":\"qidongxian|5690\",\"衡东\":\"hengdong|5693\",\"冷水江\":\"lengshuijiangshi|9470\",\"涟源\":\"lianyuanshi|9471\",\"双峰\":\"shuangfengxian|9473\",\n" + + " \"邵阳县\":\"shaoyangxian|6955\",\"邵东\":\"shaodongxian|6954\",\"沅江\":\"yuanjiangs|10201\",\"南县\":\"nanxian|10202\",\"祁阳\":\"qiyang|8532\",\n" + + " \"湘阴\":\"xiangyin|828\",\"华容\":\"huarong|830\",\"慈利\":\"cilixian|6791\",\"攸县\":\"zzyouxian|1095\"\n" + + " }\n" + + " ,\n" + + " \"河北\":{\n" + + " \"石家庄\":\"sjz|241\",\"保定\":\"bd|424\",\"唐山\":\"ts|276\",\"廊坊\":\"lf|772\",\"邯郸\":\"hd|572\",\n" + + " \"秦皇岛\":\"qhd|1078\",\"沧州\":\"cangzhou|652\",\"邢台\":\"xt|751\",\"衡水\":\"hs|993\",\"张家口\":\"zjk|3328\",\n" + + " \"承德\":\"chengde|6760\",\"定州\":\"dingzhou|8398\",\"馆陶\":\"gt|8706\",\"张北\":\"zhangbei|11201\",\"赵县\":\"zx|9048\",\n" + + " \"正定\":\"zd|3198\",\"迁安市\":\"qianan|284\",\"任丘\":\"renqiu|656\",\"三河\":\"sanhe|776\",\"武安\":\"wuan|577\",\n" + + " \"雄安新区\":\"xionganxinqu|111234\",\"燕郊\":\"lfyanjiao|12804\",\"涿州\":\"zhuozhou|428\",\"河间\":\"hejian|658\",\"黄骅\":\"huanghua|657\",\n" + + " \"沧县\":\"cangxian|659\",\"磁县\":\"cixian|591\",\"涉县\":\"shexian|14059\",\"霸州\":\"bazhou|775\",\"香河\":\"xianghe|5395\",\n" + + " \"固安\":\"lfguan|12803\",\"遵化市\":\"zunhua|283\",\"迁西\":\"qianxixian|7061\",\"玉田\":\"yutianxian|7060\",\"滦南\":\"luannanxian|7066\",\n" + + " \"沙河\":\"shaheshi|755\"\n" + + " }\n" + + " ,\n" + + " \"江苏\":{\n" + + " \"苏州\":\"su|5\",\"南京\":\"nj|172\",\"无锡\":\"wx|93\",\"常州\":\"cz|463\",\"徐州\":\"xz|471\",\n" + + " \"南通\":\"nt|394\",\"扬州\":\"yz|637\",\"盐城\":\"yancheng|613\",\"淮安\":\"ha|968\",\"连云港\":\"lyg|2049\",\n" + + " \"泰州\":\"taizhou|693\",\"宿迁\":\"suqian|2350\",\"镇江\":\"zj|645\",\"沭阳\":\"shuyang|5772\",\"大丰\":\"dafeng|11279\",\n" + + " \"如皋\":\"rugao|397\",\"启东\":\"qidong|400\",\"溧阳\":\"liyang|469\",\"海门\":\"haimen|399\",\"东海\":\"donghai|2147\",\n" + + " \"扬中\":\"yangzhong|649\",\"兴化\":\"xinghuashi|699\",\"新沂\":\"xinyishi|478\",\"泰兴\":\"taixing|696\",\"如东\":\"rudong|402\",\n" + + " \"邳州\":\"pizhou|477\",\"沛县\":\"xzpeixian|11349\",\"靖江\":\"jingjiang|698\",\"建湖\":\"jianhu|618\",\"海安\":\"haian|401\",\n" + + " \"东台\":\"dongtai|615\",\"丹阳\":\"danyang|648\",\"宝应县\":\"baoyingx|14451\",\"灌南\":\"guannan|2150\",\"灌云\":\"guanyun|2148\",\n" + + " \"姜堰\":\"jiangyan|697\",\"金坛\":\"jintan|468\",\"昆山\":\"szkunshan|16\",\"泗洪\":\"sihong|5958\",\"泗阳\":\"siyang|5959\",\n" + + " \"句容\":\"jurong|650\",\"射阳\":\"sheyang|621\",\"阜宁\":\"funingxian|620\",\"响水\":\"xiangshui|619\",\"盱眙\":\"xuyi|976\",\n" + + " \"金湖\":\"jinhu|975\",\"江阴\":\"jiangyins|34984\"\n" + + " }\n" + + " ,\n" + + " \"江西\":{\n" + + " \"南昌\":\"nc|669\",\"赣州\":\"ganzhou|2363\",\"九江\":\"jj|2247\",\"宜春\":\"yichun|5709\",\"吉安\":\"ja|2364\",\n" + + " \"上饶\":\"sr|10120\",\"萍乡\":\"px|2248\",\"抚州\":\"fuzhou|10134\",\"景德镇\":\"jdz|2360\",\"新余\":\"xinyu|10115\",\n" + + " \"鹰潭\":\"yingtan|3209\",\"永新\":\"yxx|11077\",\"乐平\":\"lepingshi|9048\",\"进贤\":\"jinxian|677\",\"分宜\":\"fenyi|10118\",\n" + + " \"丰城\":\"fengchengshi|5711\",\"樟树\":\"zhangshu|5713\",\"高安\":\"gaoan|5712\",\"余江\":\"yujiang|3210\",\"南城\":\"nanchengx|10137\",\n" + + " \"浮梁\":\"fuliangxian|9071\"\n" + + " }\n" + + " ,\n" + + " \"吉林\":{\n" + + " \"长春\":\"cc|319\",\"吉林\":\"jl|700\",\"四平\":\"sp|10171\",\"延边\":\"yanbian|3184\",\"松原\":\"songyuan|2315\",\n" + + " \"白城\":\"bc|5918\",\"通化\":\"th|10159\",\"白山\":\"baishan|10179\",\"辽源\":\"liaoyuan|2501\",\"公主岭\":\"gongzhuling|10171\",\n" + + " \"梅河口\":\"meihekou|10162\",\"扶余\":\"fuyuxian|9085\",\"长岭\":\"changlingxian|9084\",\"桦甸\":\"huadian|706\",\"磐石\":\"panshi|708\",\n" + + " \"梨树县\":\"lishu|10176\"\n" + + " }\n" + + " ,\n" + + " \"辽宁\":{\n" + + " \"沈阳\":\"sy|188\",\"大连\":\"dl|147\",\"鞍山\":\"as|523\",\"锦州\":\"jinzhou|2354\",\"抚顺\":\"fushun|5722\",\n" + + " \"营口\":\"yk|5898\",\"盘锦\":\"pj|2041\",\"朝阳\":\"cy|10106\",\"丹东\":\"dandong|3445\",\"辽阳\":\"liaoyang|2038\",\n" + + " \"本溪\":\"benxi|5845\",\"葫芦岛\":\"hld|10088\",\"铁岭\":\"tl|6729\",\"阜新\":\"fx|10097\",\"庄河\":\"pld|3306\",\n" + + " \"瓦房店\":\"wfd|3279\",\"灯塔\":\"dengta|2071\",\"凤城\":\"fengcheng|3450\",\"北票\":\"beipiao|10109\",\"开原\":\"kaiyuan|6733\"\n" + + " }\n" + + " ,\n" + + " \"宁夏\":{\n" + + " \"银川\":\"yinchuan|2054\",\"吴忠\":\"wuzhong|9962\",\"石嘴山\":\"szs|9971\",\"中卫\":\"zw|9951\",\"固原\":\"guyuan|2421\"\n" + + " }\n" + + " ,\n" + + " \"内蒙古\":{\n" + + " \"呼和浩特\":\"hu|811\",\"包头\":\"bt|801\",\"赤峰\":\"chifeng|6700\",\"鄂尔多斯\":\"erds|2037\",\"通辽\":\"tongliao|10015\",\n" + + " \"呼伦贝尔\":\"hlbe|10039\",\"巴彦淖尔市\":\"bycem|10070\",\"乌兰察布\":\"wlcb|9993\",\"锡林郭勒\":\"xl|2408\",\"兴安盟\":\"xam|9976\",\n" + + " \"乌海\":\"wuhai|2404\",\"阿拉善盟\":\"alsm|10083\",\"海拉尔\":\"hlr|2043\"\n" + + " }\n" + + " ,\n" + + " \"青海\":{\n" + + " \"西宁\":\"xn|2052\",\"海西\":\"hx|9902\",\"海北\":\"haibei|9917\",\"果洛\":\"guoluo|9936\",\"海东\":\"haidong|9909\",\n" + + " \"黄南\":\"huangnan|9896\",\"玉树\":\"ys|9888\",\"海南\":\"hainan|10574\",\"格尔木\":\"geermushi|9904\"\n" + + " }\n" + + " ,\n" + + " \"山东\":{\n" + + " \"青岛\":\"qd|122\",\"济南\":\"jn|265\",\"烟台\":\"yt|228\",\"潍坊\":\"wf|362\",\"临沂\":\"linyi|505\",\n" + + " \"淄博\":\"zb|385\",\"济宁\":\"jining|450\",\"泰安\":\"ta|686\",\"聊城\":\"lc|882\",\"威海\":\"weihai|518\",\n" + + " \"枣庄\":\"zaozhuang|961\",\"德州\":\"dz|728\",\"日照\":\"rizhao|3177\",\"东营\":\"dy|623\",\"菏泽\":\"heze|5632\",\n" + + " \"滨州\":\"bz|944\",\"莱芜\":\"lw|2292\",\"章丘\":\"zhangqiu|8680\",\"垦利\":\"kl|11313\",\"诸城\":\"zc|9146\",\n" + + " \"寿光\":\"shouguang|369\",\"龙口\":\"longkou|233\",\"曹县\":\"caoxian|5638\",\"单县\":\"shanxian|5636\",\"肥城\":\"feicheng|690\",\n" + + " \"高密\":\"gaomi|371\",\"广饶\":\"guangrao|627\",\"桓台\":\"huantaixian|7335\",\"莒县\":\"juxian|3180\",\"莱州\":\"laizhou|235\",\n" + + " \"蓬莱\":\"penglai|237\",\"青州\":\"qingzhou|367\",\"荣成\":\"rongcheng|522\",\"乳山\":\"rushan|520\",\"滕州\":\"tengzhou|967\",\n" + + " \"新泰\":\"xintai|689\",\"招远\":\"zhaoyuan|3325\",\"邹城\":\"zoucheng|455\",\"邹平\":\"zouping|946\",\"临清\":\"linqing|884\",\n" + + " \"茌平\":\"chiping|887\",\"郓城\":\"hzyc|5637\",\"博兴\":\"boxing|949\",\"东明\":\"dongming|5641\",\"巨野\":\"juye|5640\",\n" + + " \"无棣\":\"wudi|951\",\"齐河\":\"qihe|734\",\"微山\":\"weishan|459\",\"禹城\":\"yuchengshi|731\",\"临邑\":\"linyixianq|739\",\n" + + " \"乐陵\":\"leling|730\",\"莱阳\":\"laiyang|234\",\"宁津\":\"ningjin|733\",\"高唐\":\"gaotang|885\",\"莘县\":\"shenxian|888\",\n" + + " \"阳谷\":\"yanggu|886\",\"冠县\":\"guanxian|890\",\"平邑\":\"pingyi|514\",\"郯城\":\"tancheng|510\",\"沂源\":\"yiyuanxian|7334\",\n" + + " \"汶上\":\"wenshang|460\",\"梁山\":\"liangshanx|462\",\"利津\":\"lijin|628\",\"沂南\":\"yinanxian|7301\",\"栖霞\":\"qixia|238\",\n" + + " \"宁阳\":\"ningyang|691\",\"东平\":\"dongping|692\",\"昌邑\":\"changyishi|372\",\"安丘\":\"anqiu|370\",\"昌乐\":\"changle|373\",\n" + + " \"临朐\":\"linqu|374\",\"鄄城\":\"juancheng|5635\"\n" + + " }\n" + + " ,\n" + + " \"山西\":{\n" + + " \"太原\":\"ty|740\",\"临汾\":\"linfen|5669\",\"大同\":\"dt|6964\",\"运城\":\"yuncheng|5653\",\"晋中\":\"jz|8854\",\n" + + " \"长治\":\"changzhi|6921\",\"晋城\":\"jincheng|3350\",\"阳泉\":\"yq|8760\",\"吕梁\":\"lvliang|3222\",\"忻州\":\"xinzhou|3453\",\n" + + " \"朔州\":\"shuozhou|9871\",\"临猗\":\"linyixian|9193\",\"清徐\":\"qingxu|10908\",\"柳林\":\"liulin|3225\",\"高平\":\"gaoping|3354\",\n" + + " \"泽州\":\"zezhou|3353\",\"襄垣\":\"xiangyuanxian|6928\",\"孝义\":\"xiaoyi|3227\"\n" + + " }\n" + + " ,\n" + + " \"陕西\":{\n" + + " \"西安\":\"xa|483\",\"咸阳\":\"xianyang|7453\",\"宝鸡\":\"baoji|2044\",\"渭南\":\"wn|5733\",\"汉中\":\"hanzhong|3163\",\n" + + " \"榆林\":\"yl|5942\",\"延安\":\"yanan|8973\",\"安康\":\"ankang|3157\",\"商洛\":\"sl|9854\",\"铜川\":\"tc|9832\",\n" + + " \"神木\":\"shenmu|5944\",\"韩城\":\"hancheng|5735\",\"府谷\":\"fugu|5945\",\"靖边\":\"jingbian|5947\",\"定边\":\"dingbian|5948\"\n" + + " }\n" + + " ,\n" + + " \"四川\":{\n" + + " \"成都\":\"cd|102\",\"绵阳\":\"mianyang|1057\",\"德阳\":\"deyang|2373\",\"南充\":\"nanchong|2378\",\"宜宾\":\"yb|2380\",\n" + + " \"自贡\":\"zg|6745\",\"乐山\":\"ls|3237\",\"泸州\":\"luzhou|2372\",\"达州\":\"dazhou|9799\",\"内江\":\"scnj|5928\",\n" + + " \"遂宁\":\"suining|9688\",\"攀枝花\":\"panzhihua|2371\",\"眉山\":\"ms|9704\",\"广安\":\"ga|2381\",\"资阳\":\"zy|6803\",\n" + + " \"凉山\":\"liangshan|9717\",\"广元\":\"guangyuan|9749\",\"雅安\":\"ya|9687\",\"巴中\":\"bazhong|9811\",\"阿坝\":\"ab|9817\",\n" + + " \"甘孜\":\"ganzi|9764\",\"安岳\":\"anyuexian|6806\",\"广汉\":\"guanghanshi|8719\",\"简阳\":\"jianyangshi|6805\",\"仁寿\":\"renshouxian|9706\",\n" + + " \"射洪\":\"shehongxian|9694\",\"大竹\":\"dazu|9806\",\"宣汉\":\"xuanhan|9804\",\"渠县\":\"qux|9807\",\"长宁\":\"changningx|7148\"\n" + + " }\n" + + " ,\n" + + " \"新疆\":{\n" + + " \"乌鲁木齐\":\"xj|984\",\"昌吉\":\"changji|8582\",\"巴音郭楞\":\"bygl|9530\",\"伊犁\":\"yili|9472\",\"阿克苏\":\"aks|9499\",\n" + + " \"喀什\":\"ks|9326\",\"哈密\":\"hami|7452\",\"克拉玛依\":\"klmy|2042\",\"博尔塔拉\":\"betl|9529\",\"吐鲁番\":\"tlf|9475\",\n" + + " \"和田\":\"ht|9489\",\"石河子\":\"shz|9551\",\"克孜勒苏\":\"kzls|9519\",\"阿拉尔\":\"ale|9539\",\"五家渠\":\"wjq|9562\",\n" + + " \"图木舒克\":\"tmsk|9559\",\"库尔勒\":\"kel|7168\",\"阿勒泰\":\"alt|18837\",\"塔城\":\"tac|18845\"\n" + + " }\n" + + " ,\n" + + " \"西藏\":{\n" + + " \"拉萨\":\"lasa|2055\",\"日喀则\":\"rkz|9615\",\"山南\":\"sn|9576\",\"林芝\":\"linzhi|9646\",\"昌都\":\"changdu|9648\",\n" + + " \"那曲\":\"nq|9618\",\"阿里\":\"al|9678\",\"日土\":\"rituxian|9682\",\"改则\":\"gaizexian|9684\"\n" + + " }\n" + + " ,\n" + + " \"云南\":{\n" + + " \"昆明\":\"km|541\",\"曲靖\":\"qj|2389\",\"大理\":\"dali|2398\",\"红河\":\"honghe|2394\",\"玉溪\":\"yx|2040\",\n" + + " \"丽江\":\"lj|2392\",\"文山\":\"ws|2395\",\"楚雄\":\"cx|2393\",\"西双版纳\":\"bn|2397\",\"昭通\":\"zt|9409\",\n" + + " \"德宏\":\"dh|9437\",\"普洱\":\"pe|9444\",\"保山\":\"bs|2390\",\"临沧\":\"lincang|9422\",\"迪庆\":\"diqing|9432\",\n" + + " \"怒江\":\"nujiang|9462\",\"弥勒\":\"milexian|8892\",\"安宁\":\"anningshi|547\",\"宣威\":\"xuanwushi|7533\"\n" + + " }\n" + + " ,\n" + + " \"浙江\":{\n" + + " \"杭州\":\"hz|79\",\"宁波\":\"nb|135\",\"温州\":\"wz|330\",\"金华\":\"jh|531\",\"嘉兴\":\"jx|497\",\n" + + " \"台州\":\"tz|403\",\"绍兴\":\"sx|355\",\"湖州\":\"huzhou|831\",\"丽水\":\"lishui|7921\",\"衢州\":\"quzhou|6793\",\n" + + " \"舟山\":\"zhoushan|8481\",\"乐清\":\"yueqingcity|13950\",\"瑞安\":\"ruiancity|13951\",\"义乌\":\"yiwu|12291\",\"余姚\":\"yuyao|5333\",\n" + + " \"诸暨\":\"zhuji|357\",\"象山\":\"xiangshanxian|6738\",\"温岭\":\"wenling|408\",\"桐乡\":\"tongxiang|502\",\"慈溪\":\"cixi|5334\",\n" + + " \"长兴\":\"changxing|834\",\"嘉善\":\"jiashanx|14357\",\"海宁\":\"haining|500\",\"德清\":\"deqing|835\",\"东阳\":\"dongyang|536\",\n" + + " \"安吉\":\"anji|836\",\"苍南\":\"cangnanxian|7579\",\"临海\":\"linhai|407\",\"永康\":\"yongkang|537\",\"玉环\":\"yuhuan|409\",\n" + + " \"平湖\":\"pinghushi|501\",\"海盐\":\"haiyan|504\",\"武义县\":\"wuyix|14528\",\"嵊州\":\"shengzhou|359\",\"新昌\":\"xinchang|361\",\n" + + " \"江山\":\"jiangshanshi|6796\",\"平阳\":\"pingyangxian|7575\"\n" + + " }\n" + + " ,\n" + + " \"其他\":{\n" + + " \"中国香港\":\"hk|2050\",\"中国澳门\":\"am|9399\",\"中国台湾\":\"tw|2051\"\n" + + " }\n" + + " ,\n" + + " \"海外\":{\n" + + " \"洛杉矶\":\"gllosangeles\",\"旧金山\":\"glsanfrancisco\",\"纽约\":\"glnewyork\",\"多伦多\":\"gltoronto\",\"温哥华\":\"glvancouver\",\n" + + " \"伦敦\":\"glgreaterlondon\",\"莫斯科\":\"glmoscow\",\"首尔\":\"glseoul\",\"东京\":\"gltokyo\",\"新加坡\":\"glsingapore\",\n" + + " \"曼谷\":\"glbangkok\",\"清迈\":\"glchiangmai\",\"迪拜\":\"gldubai\",\"奥克兰\":\"glauckland\",\"悉尼\":\"glsydney\",\n" + + " \"墨尔本\":\"glmelbourne\",\"其他海外城市\":\"city\"\n" + + " }\n" + + " }"; + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + JSONObject jsonObject = JSONObject.parseObject(cityList) + Set provinceSet = jsonObject.keySet() + List ret = new ArrayList<>() + + for (String province: provinceSet) { + JSONObject provinceInfo = jsonObject[province] + for (String city : provinceInfo.keySet()) { + // 屏蔽县 + if (citySet.contains(city)) { + def city_short = provinceInfo[city].split("\\|")[0] + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", city_short) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "58.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + print("58城市数据写入成功") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy new file mode 100644 index 000000000..caf289056 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.time.DateUtils + +/** + * @author shenjian + * @since 2021/4/30 + */ +class Comm58 { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static final def source = "58同城" + static final def root = "58.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("city","福州") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(10000).each { obj -> + try { + def url = obj.district_id + def cityShort = "fz" + + String[] info = url.split("/") + def id = info[info.size() - 1] + if (!set.contains(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + if (obj.region != null) { + set.add(id) + doc.add(dbObject) + } + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } catch(Exception ex) { + println("解析异常:" + obj) + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy new file mode 100644 index 000000000..ca8e4379c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.crawl.InvokeHttpRequest +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +/** + * 统计58小区总量 + * + * @author shenjian + * @since 2021/5/26 + */ +class Count58Seaweed { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + + static void main(String[] args) { + crawlSeaweedCount() + println(count()) + } + + static crawlSeaweedCount() { + cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { obj -> + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".58.com/xiaoqu/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + String str = JQueryUtils.selectText(".total-info", doc) + // 117.60.236.41 + if (str) { + str = str.replace("共找到", "").replace("个小区", "").trim() + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count))) + println("当前处理城市" + obj.city) + } + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + } + + static count() { + int number = 0; + cityConn.find(new BasicDBObject("root", "58.com"), new BasicDBObject("district_count", 1)) + .batchSize(100).each { obj -> + if (obj.district_count) { + number += obj.district_count + } + } + return number + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy new file mode 100644 index 000000000..d02c980c1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy @@ -0,0 +1,73 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 58小区详情爬取 + * + * @author shenjian* @since 2021/5/26 + */ +@Slf4j +class District58DetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(100000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy new file mode 100644 index 000000000..79bf00e9a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 58有效种子梳理 + * + * @author shenjian + * @since 2021/5/14 + */ +class Seaweed58 { + + static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static void main(String[] args) { + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + + DBObject query = new BasicDBObject() + query.put("root", "58.com") + + DBObject filter = new BasicDBObject() + filter.put("city", "1") + filter.put("_id", "1") + def delIds = [] + districtSeedsConn.find(query, filter).batchSize(20000).each {obj -> + if (!citySet.contains(obj.city)) { + delIds.add(obj._id) + if (delIds.size() == 10000) { + districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds))) + delIds = [] + } + } + } + districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds))) + delIds = [] + print("58种子清理成功") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy new file mode 100644 index 000000000..46659dc54 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -0,0 +1,56 @@ +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateUtils + +/** + * 房天下小区种子生成 + */ +class CommFang { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source","搜房") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(1000).each { obj -> + def url = obj.district_id + def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf(".")) + def id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf(".")) + if (!set.contains(id) && StringUtils.isNumeric(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", "fang.com") + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + if (obj.region != null) { + set.add(id) + doc.add(dbObject) + } + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy new file mode 100644 index 000000000..0cf014627 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy @@ -0,0 +1,57 @@ +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateUtils + +/** + * 贝壳找房小区种子生成 + */ +class CommKe { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static final def source = "贝壳找房" + static final def root = "ke.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("url", 1) + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(1000).each { obj -> + def url = obj.url + def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf(".")) + def id = obj.district_id + if (!set.contains(id) && StringUtils.isNumeric(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + set.add(id) + doc.add(dbObject) + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy new file mode 100644 index 000000000..56a389f28 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy @@ -0,0 +1,33 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 种子数据备份 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeedBak { + + static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static DBCollection aimConn = MongoUtils.getCollectionByName("seeddb_dev", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + List list = new ArrayList<>() + + districtSeedsConn.find().batchSize(20000).each {obj -> + list.add(obj) + if (list.size() == 20000) { + aimConn.insert(list) + list.clear() + } + } + aimConn.insert(list) + list.clear() + println("种子备份成功") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy new file mode 100644 index 000000000..c9caccb1a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian.crawl.anjuke + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 安居客小区详情抓取 + * + * @author shenjian + * @since 2021/5/28 + */ +@Slf4j +class AnjukeDistrictDetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "anjuke.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".anjuke.com/community/view/" + obj.district_id + "#/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("请拖动滑块进行验证") == -1 && content.indexOf("访问验证-安居客") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_anjuke") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } else if (res && res.code == 508) { + log.info("508 {}", obj.district_id) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy new file mode 100644 index 000000000..88a3b6f38 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * 贝壳找房城市种子生成 + * + * @author shenjian + * @since 2021/6/2 + */ +class CityFang { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + // 房天下较为特殊,直接从url取乱码,需要改页面所有非UTF-8编码为UTF-8 + Document doc = Jsoup.parse(new File("F:\\数据\\爬虫\\fang_city.html"), "UTF-8"); + Elements elements = JQueryUtils.select(".outCont li", doc) + List ret = new ArrayList<>() + for (Element element : elements) { + String province = JQueryUtils.selectText("strong", element) + Elements cityElements = JQueryUtils.select("a", element) + for (Element cityElement: cityElements) { + String city = cityElement.text() + if ("直辖市".equals(province)) { + province = city + } + if (citySet.contains(city)) { + String url = cityElement.attr("href") + String cityShort = StringUtils.substringBetween(url,"//", ".esf.fang.com") + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", cityShort) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "fang.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + println("房天下城市数据写入成功") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy new file mode 100644 index 000000000..e8ae7fc0c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy @@ -0,0 +1,57 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * 贝壳找房城市种子生成 + * + * @author shenjian + * @since 2021/6/2 + */ +class CityKe { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + final def URL = "https://www.ke.com/city/" + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + Elements elements = JQueryUtils.select(".city_list_ul .city_list_li", doc) + List ret = new ArrayList<>() + for (Element element : elements) { + String province = JQueryUtils.selectText(".city_province .city_list_tit", element) + Elements cityElements = JQueryUtils.select(".CLICKDATA a", element) + for (Element cityElement: cityElements) { + String city = cityElement.text() + if (citySet.contains(city)) { + String url = cityElement.attr("href") + String cityShort = StringUtils.substringBetween(url,"//", ".ke.com") + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", cityShort) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "ke.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + println("贝壳找房城市数据写入成功") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy new file mode 100644 index 000000000..d8b2c466b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy @@ -0,0 +1,100 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 贝壳各城市成交总量抓取 + * + * @author shenjian + * @since 2021/6/2 + */ +class CountTradeKe { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + //crawlTradeCount() + //println(count()) + //1896919 1760177 + findUnNormalCount() + } + + static crawlTradeCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { objTemp -> + final def obj = objTemp + executorService.execute(new Runnable() { + @Override + void run() { + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".ke.com/chengjiao/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + boolean existTrade = JQueryUtils.selectText(".resultDes .total", doc).contains("成交房源") + if (existTrade) { + String str = JQueryUtils.selectText(".resultDes .total span", doc) + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", count))) + } else { + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", 0))) + } + println("当前处理城市" + obj.city) + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + }) + } + + executorService.shutdown() + } + + /** + * 找出异常抓取的城市 + */ + static findUnNormalCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city", 1).append("city_short", 1).append("trade_count", 1)) + .batchSize(100).each { obj -> + Long realTradeCount = obj.trade_count + if (realTradeCount > 0) { + Long crawledCount = dealPriceConn.count(new BasicDBObject("city", obj.city).append("source", "贝壳找房")) + if (realTradeCount > crawledCount) { + println(obj.city + " " + obj.city_short + "抓取异常:抓取为" + crawledCount + " 实际为" + realTradeCount + " 少抓" + (realTradeCount - crawledCount)) + } + } + } + } + + /** + * 统计实际抓取量 + * + * @return + */ + static count() { + int number = 0; + cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1)) + .batchSize(100).each { obj -> + if (obj.trade_count) { + number += obj.trade_count + } + } + return number + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy new file mode 100644 index 000000000..6cf7730ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/didata/MatchFromExcel.groovy @@ -0,0 +1,28 @@ +package com.fangjia.script.temp.shenjian.didata + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient + +/** + * 从excel中匹配出 + */ +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\地数\\数据建设\\待审核小区(地图匹配).xlsx", { row -> + def city = row['A'] + def region = row['B'] + def district_name = row['C'] + + def gaodeRegion = row['H'] + def gaodeName = row['I'] + + def jsonObject = OpenApiClient.addressMatch(city, district_name, "district", false, region, district_name, "strict") + if (jsonObject != null && jsonObject.precise != -1) { + row['D'] = jsonObject.get("name") + } + + if (gaodeRegion && gaodeName) { + def jsonObject2 = OpenApiClient.addressMatch(city, gaodeName, "district", false, gaodeRegion, gaodeName, "strict") + if (jsonObject2 != null && jsonObject2.precise != -1) { + row['J'] = jsonObject2.get("name") + } + } +}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy new file mode 100644 index 000000000..e308339de --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.temp.shenjian.dingding + +import com.fangjia.script.util.DateUtils + + +/** + * 钉钉接口调用工具类 + * + * @author shenjian + * @since 2021/5/8 + */ +class DingTalkUtils { + + static main(String[] args) { + // 要导出的月份第一天 + Date now = DateUtils.parseFromTightByPattern("2022-03-01", "yyyy-MM-dd") ; + + String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" + String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") + String endTime = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(now), "yyyy-MM-dd") + " 23:59:59" + ScoreExport.export(startTime, monthStartTime, endTime) + ProjectExport.export(startTime, monthStartTime, endTime) + TaskExport.export(startTime, monthStartTime, endTime) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy new file mode 100644 index 000000000..b2535679e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy @@ -0,0 +1,110 @@ +package com.fangjia.script.temp.shenjian.dingding + +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import org.apache.commons.collections.CollectionUtils + +/** + * 项目导出 + * + * @author shenjian + * @since 2021/5/8 + */ +class ProjectExport { + + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" + + static main(args) { + export(startTime, monthStartTime, endTime) + } + + static export(def startTime, def monthStartTime, def endTime) { + List> mergeData = new ArrayList<>() + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def result = [] + + List projects = ApiUtil.listProject(); + for (Project project : projects) { + def ret = [] + List labels = ApiUtil.listProjectLabel(project.getId()) + ret[1] = project.getName() + for (ProjectLabel label : labels) { + if ("商务负责人" == label.getGroups().getName()) { + if (ret[2] == null) { + ret[2] = label.getName() + } else { + ret[2] = ret[2] + "、" + label.getName() + } + } else if ("项目类别" == label.getGroups().getName()) { + ret[0] = label.getName() + } + } + // 获取项目总工时 + int limitStart = 0 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + List tempTasks = tasks; + + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { + limitStart += 50 + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } + } + Double totalTime = 0D; + if (CollectionUtils.isNotEmpty(tasks)) { + for (Task task: tasks) { + // 获取计划时间在当月的任务 + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) + if (taskDetail.getFlowConten() != null) { + totalTime += Double.valueOf(taskDetail.getFlowConten()) + } + } + } + ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, totalTime) + result << ret + } + def sortedResult = [] + sortedResult << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] + for (def label : sort) { + List mergeRowCel = new ArrayList<>() + mergeRowCel.add(sortedResult.size()) + for (def ret : result) { + if (ret[0] == label) { + sortedResult << ret + } + } + mergeRowCel.add(sortedResult.size() - 1) + mergeRowCel.add(0) + mergeRowCel.add(0) + + mergeData.add(mergeRowCel) + } + + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计", mergeData); + } +} + + + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy new file mode 100644 index 000000000..2218e2ec6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy @@ -0,0 +1,154 @@ +package com.fangjia.script.temp.shenjian.dingding + +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import org.apache.commons.collections.CollectionUtils + +/** + * 绩效导出 + * + * @author shenjian + * @since 2021/5/10 + */ +public class ScoreExport { + + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" + + static main(args) { + export(startTime, monthStartTime, endTime) + } + + static void export(def startTime, def monthStartTime, def endTime) { + List> mergeData = new ArrayList<>() + def sortedResult = [] + sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def result = [] + List projects = ApiUtil.listProject(); + for (Project project : projects) { + def ret = [] + List labels = ApiUtil.listProjectLabel(project.getId()) + ret[2] = project.getName() + for (ProjectLabel label : labels) { + if ("商务负责人" == label.getGroups().getName()) { + if (ret[3] == null) { + ret[3] = label.getName() + } else { + ret[3] = ret[3] + "、" + label.getName() + } + } else if ("项目类别" == label.getGroups().getName()) { + ret[0] = label.getName() + } + } + // 获取项目下任务 + int limitStart = 0 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + List tempTasks = tasks; + + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { + limitStart += 50 + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } + } + if (CollectionUtils.isNotEmpty(tasks)) { + for (Task task: tasks) { + // 获取计划时间在当月的任务 + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { + boolean find = false + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true + } + } + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[sortedResult[0].size() - 1] = Double.valueOf(taskDetail.getFlowConten()) + } + } + } + } + result << ret + } + def allTotal = [] + for (def label : sort) { + def total = [] + int startRow = sortedResult.size() + for (def ret : result) { + if (ret[0] == label) { + sortedResult << ret + for (int i = 4; i < ret.size(); i++) { + if (total[i] == null) { + total[i] = ret[i] == null ? 0 : Double.valueOf(ret[i]) + } else { + total[i] += (ret[i] == null ? 0 : Double.valueOf(ret[i])) + } + } + } + } + int endRow = sortedResult.size() - 1 + List mergeRowCel = new ArrayList<>() + mergeRowCel.add(startRow) + mergeRowCel.add(endRow) + mergeRowCel.add(0) + mergeRowCel.add(0) + mergeData.add(mergeRowCel) + + mergeRowCel = new ArrayList<>() + mergeRowCel.add(startRow) + mergeRowCel.add(endRow) + mergeRowCel.add(1) + mergeRowCel.add(1) + mergeData.add(mergeRowCel) + + double totalNow = 0 + total[2] = "小计" + sortedResult << total + for (int i = 4; i < total.size(); i++) { + if (allTotal[i] == null) { + allTotal[i] = total[i] == null ? 0 : Double.valueOf(total[i]) + } else { + allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + totalNow += total[i] == null ? 0 : Double.valueOf(total[i]) + } + sortedResult[startRow][1] = totalNow + } + allTotal[2] = "人员绩效总计" + sortedResult << allTotal + + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", sortedResult, "绩效统计", mergeData); + } +} + + + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy new file mode 100644 index 000000000..d7958a211 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -0,0 +1,134 @@ +package com.fangjia.script.temp.shenjian.dingding + +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import com.fangjia.script.statistics.TaskPersonDTO +import org.apache.commons.collections.CollectionUtils + +/** + * 任务统计 + * + * @author shenjian + * @since 2021/5/10 + */ +public class TaskExport { + + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" + + static main(args) { + export(startTime, monthStartTime, endTime) + } + + static void export(def startTime, def monthStartTime, def endTime) { + List> mergeData = new ArrayList<>() + List list = new ArrayList<>() + def result = [] + result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] + int limitStart = 0 + List tasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + List tempTasks = tasks; + + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { + limitStart += 50 + tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } + } + for (Task task: tasks) { + // 获取计划时间在当月的任务 + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { + TaskPersonDTO dto = new TaskPersonDTO() + dto.name = task.response.name + dto.project = task.project.name + dto.taskName = task.name + dto.score = org.apache.commons.lang3.StringUtils.isBlank(taskDetail.flowConten) ? 0d : Double.valueOf(taskDetail.flowConten) + dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) + + list.add(dto) + } + } + + Collections.sort(list) + + Map> map = new HashMap<>() + for (TaskPersonDTO dto : list) { + def ret = [] + ret[0] = dto.name; + ret[1] = dto.project; + ret[2] = dto.taskName; + ret[3] = dto.score; + ret[4] = dto.workTime; + if (map.containsKey(dto.name)) { + Map info = map.get(dto.name) + info.put("endRow", Double.valueOf(info.get("endRow") + 1)) + info.put("totalScore", Double.valueOf(info.get("totalScore") + dto.score)) + info.put("workTime", Double.valueOf(info.get("workTime") + dto.workTime)) + map.put(dto.name, info); + } else { + if (result.size() > 1) { + def total = [] + total[2] = "小计" + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + result << total + } + Map info = new HashMap<>() + info.put("startRow", Double.valueOf(result.size())) + info.put("endRow", Double.valueOf(result.size())) + info.put("startCell", 0D) + info.put("endCell", 0D) + info.put("totalScore", Double.valueOf(dto.score)) + info.put("workTime", Double.valueOf(dto.workTime)) + map.put(dto.name, info) + } + result << ret + } + def total = [] + total[2] = "小计" + if (map.get(result[result.size() - 1][0]) != null) { + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + } else { + total[3] = 0 + total[4] = 0 + print("无该信息" + result[result.size() - 1][0]) + } + result << total + + for (Map info : map.values()) { + List data = new ArrayList<>() + data.add(info.get("startRow").toInteger()) + data.add(info.get("endRow").toInteger()) + data.add(info.get("startCell").toInteger()) + data.add(info.get("endCell").toInteger()) + mergeData.add(data) + } + + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", result, "任务统计", mergeData); + } +} + + + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy new file mode 100644 index 000000000..39ed00ec4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.temp.shenjian.dto + +/** + * @author shenjian + * @since 2021/5/18 + */ +class AroundDTO implements Comparable { + + String id; + String name; + String type + Integer distance + + @Override + public int compareTo(AroundDTO o2) { + if (this.type.equals(o2.type)) { + if (this.distance != null) { + return this.distance.compareTo(o2.distance); + } + } else { + return this.type.compareTo(o2.type); + } + return 0; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy new file mode 100644 index 000000000..01b1c1f27 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy @@ -0,0 +1,16 @@ +package com.fangjia.script.temp.shenjian.dto + +/** + * 匹配返回结构体 + */ +class SearchAddressDTO { + + String code; + String msg; + Result result; + + class Result { + String street + String streetNumber + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy new file mode 100644 index 000000000..399b33c38 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy @@ -0,0 +1,124 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import org.apache.commons.lang3.time.DateUtils +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.* +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.index.reindex.DeleteByQueryRequest +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder + +import java.util.concurrent.atomic.AtomicLong + +/** + * 拷贝1.31前四周数据至3.28日缺失的四周数据中 + * 注意时间更新 + * + * @author shenjian + * @since 2021/3/30 + */ +class CopyDataForAnnPricePre { + + static RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient() + + static final long timestamp = (DateUtils.parseDate("2021-03-28", "yyyy-MM-dd").getTime() - + DateUtils.parseDate("2021-01-31", "yyyy-MM-dd").getTime()) + static final int days = timestamp / 1000 / 3600 / 24 + + static void main(String[] args) { + // 验证数据是否正确写入 + //String[] weeksNew= ["2021-03-07", "2021-03-14", "2021-03-21", "2021-03-28"] + String[] weeksNew= ["2021-03-07", "2021-01-31"] + for (String week : weeksNew) { + println(DateUtils.parseDate(week, "yyyy-MM-dd").getTime()) + } + //String[] weeksNew=["2021-03-28"] +// ///////////////////// 删除数据临时逻辑 +// // 先删除数据 +// DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("ann_price_pre") +// deleteRequest.setQuery(QueryBuilders.termQuery("date", DateUtils.parseDate("2021-03-07", "yyyy-MM-dd").getTime())) +// // 批次大小 +// deleteRequest.setBatchSize(1000); +// restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT) +// ///////////////////// + checkNum(weeksNew) + + BulkRequest request = new BulkRequest(); + for (String week : weeksNew) { + AtomicLong count = new AtomicLong(0) + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.addDays(DateUtils.parseDate(week, "yyyy-MM-dd"), -days).getTime())) + .size(10000); + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); + SearchRequest searchRequest = new SearchRequest("ann_price_pre") + .source(searchSourceBuilder) + .scroll(scroll); // 设置scroll间隔 + + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); //通过发送初始搜索请求来初始化搜索上下文 + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits() + write(count, searchHits, timestamp, request); + + while (searchHits != null && searchHits.length > 0) { //通过循环调用搜索滚动api来检索所有搜索命中,直到没有文档返回 + //处理返回的搜索结果 + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); //创建一个新的搜索滚动请求,保存最后返回的滚动标识符和滚动间隔 + scrollRequest.scroll(scroll); + searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + write(count, searchHits, timestamp, request); + } + if (count.get() > 0) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT) + println(week + "剩余批量操作成功:" + bulkResponse.status()) + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); //完成滚动后,清除滚动上下文 + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + boolean succeeded = clearScrollResponse.isSucceeded(); + println("清除滚动句柄" + succeeded) + } + checkNum(weeksNew) + + restHighLevelClient.close() + } + + static write(def count, def searchHits, def timestamp, def request) { + for (SearchHit hit : searchHits) { + Map map = hit.getSourceAsMap(); + map.put("date", Long.valueOf(map.get("date").toString()) + timestamp) + + request.add(new IndexRequest("ann_price_pre").source(map).opType(DocWriteRequest.OpType.CREATE)); + count.incrementAndGet() + if (count.get() >= 5000) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + println("批量操作成功:" + bulkResponse.status()) + request = new BulkRequest(); + count = new AtomicLong(0) + } + } + } + + static checkNum(def weeksNew) { + // 验证数据是否正确写入 + for (String week : weeksNew) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.parseDate(week, "yyyy-MM-dd").getTime())); + + SearchRequest searchRequest = new SearchRequest("ann_price_pre") + searchRequest.source(searchSourceBuilder) + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + + println(searchResponse.status().name() + " " + searchResponse.getHits().totalHits) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy new file mode 100644 index 000000000..68d16920d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy @@ -0,0 +1,98 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +/** + * 由于快照恢复字段类型错误 + * 同步es6.4该索引至es7.11,先写入mongo + * + * @author shenjian + * @since 2021/3/30 + */ +class SyncDistrictPriceStat { + + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017) + + static void main(String[] args) { + QueryBuilder queryBuilder = QueryBuilders.matchAllQuery() + def districtPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder) + //def newDistrictPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder) +// RestHighLevelClient restHighLevelClient206 = getClient206() +// RestHighLevelClient restHighLevelClient = getClient() +// //def districtPriceStats +// GetResponse getResponse = getDocument(restHighLevelClient) + for (def map : districtPriceStats) { + DBObject dbObject = new BasicDBObject() + + dbObject.put("badcase_count", map.badcase_count) + dbObject.put("city", map.city) + dbObject.put("city_grade", map.city_grade) + dbObject.put("credit_district_count", map.credit_district_count) + dbObject.put("date", map.date) + dbObject.put("district_count", map.district_count) + dbObject.put("price_district_count", map.price_district_count) + dbObject.put("score", map.score) + dbObject.put("vote_district", map.vote_district) + + fangjiaConn.save(dbObject) + +// // 创建索引请求 +// IndexRequest request = new IndexRequest("district_price_stat") +// request.id(id).timeout(TimeValue.timeValueSeconds(1)) +// request.source(map) +// +// // 将请求发出去,获取响应结果 +// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); +// +// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE +// System.out.println(indexResponse.toString()); //响应内容 + } + //restHighLevelClient.close() + } + +// static RestHighLevelClient getClient() { +// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.216",9200,"http")) +// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { +// @Override +// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { +// httpClientBuilder.disableAuthCaching(); +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); +// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); +// return httpClientBuilder; +// } +// })) +// return restHighLevelClient +// } +// +// static RestHighLevelClient getClient206() { +// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.206",9200,"http")) +// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { +// @Override +// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { +// httpClientBuilder.disableAuthCaching(); +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); +// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); +// return httpClientBuilder; +// } +// })) +// return restHighLevelClient +// } +// +// static GetResponse getDocument(RestHighLevelClient restHighLevelClient206) throws IOException { +// SearchRequest searchRequest = new SearchRequest("district_price_stat"); +// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); +// //searchSourceBuilder.query(QueryBuilders.termQuery("url", "test")); +// searchRequest.source(searchSourceBuilder); +// SearchResponse response = restHighLevelClient206.search(searchRequest, RequestOptions.DEFAULT); +// System.out.println(response.getSourceAsString()); //获取文档的内容 +// return response +// } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy new file mode 100644 index 000000000..999f1eaf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import com.mongodb.DBCollection +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.index.reindex.DeleteByQueryRequest + +import java.util.concurrent.atomic.AtomicLong + +/** + * 由于快照恢复字段类型错误 + * 同步es6.4该索引至es7.11 + * + * @author shenjian + * @since 2021/3/30 + */ +class SyncDistrictPriceStat2 { + + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017) + + static void main(String[] args) { + RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient() + // 先删除数据 + DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("district_price_stat") + deleteRequest.setQuery(QueryBuilders.matchAllQuery()) + // 批次大小 + deleteRequest.setBatchSize(1000); + restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT) + + BulkRequest request = new BulkRequest(); + AtomicLong count = new AtomicLong() + fangjiaConn.find().batchSize(10000).each {map -> + Map dbObject = new HashMap() + dbObject.put("badcase_count", map.badcase_count) + dbObject.put("city", map.city) + dbObject.put("city_grade", map.city_grade) + dbObject.put("credit_district_count", map.credit_district_count) + dbObject.put("date", map.date) + dbObject.put("district_count", map.district_count) + dbObject.put("price_district_count", map.price_district_count) + dbObject.put("score", map.score) + dbObject.put("vote_district", map.vote_district) + + request.add(new IndexRequest("district_price_stat").source(dbObject).opType(DocWriteRequest.OpType.CREATE)); + count.incrementAndGet() + // 每一千写入 + if (count.get() >= 1000) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + request = new BulkRequest(); + count = new AtomicLong(0) + } + + // 单个插入文档 +// // 创建索引请求 +// IndexRequest request = new IndexRequest("district_price_stat") +// request.timeout(TimeValue.timeValueSeconds(1)) +// request.source(dbObject) +// +// // 将请求发出去,获取响应结果 +// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); + +// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE +// System.out.println(indexResponse.toString()); //响应内容 + } + // 不足一千写入 + if (count.get() > 0) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + } + restHighLevelClient.close() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy new file mode 100644 index 000000000..bbbe4ebf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy @@ -0,0 +1,35 @@ +package com.fangjia.script.temp.shenjian.es.util + +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient + +/** + * @author shenjian + * @since 2021/3/30 + */ +class RestHighLevelClientUtils { + + static final def HOST = "192.168.0.216" + + static RestHighLevelClient getClient() { + RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(HOST,9200,"http")) + .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + httpClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpClientBuilder; + } + })) + return restHighLevelClient + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy new file mode 100644 index 000000000..ada6e667b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.fill + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.temp.shenjian.util.DistrictUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.GeoHelper +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 根据周围500米容积率与绿化率平均值推算小区 + * + * @author shenjian* + * @since 2021-03-13 + */ +@Slf4j +class CalcCubageAndVirescence { + + static main(args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("CalcCubageAndVirescence") + log.info("开始根据周围500米容积率与绿化率平均值推算小区") + def cities = ['北京', '重庆', '哈尔滨'] + for (def city : cities) { + def sameRegionDistricts = [:] + def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts); + def result = [] + districts.each { obj -> + if (obj.finish_building_date) { + int year = CommonUtils.getBuildYear(obj.finish_building_date) + if (year > 2000) { + if (!(obj.cubage_percent && obj.virescence_percent)) { + def temp = this.findResult(obj.lat2 / 10000000000.0d, obj.lng2 / 10000000000.0d, 500, sameRegionDistricts.get(obj.region), obj.name, obj._id) + result << temp + } + + } + } + } + def headers = ["小区ID", "容积率", "绿化率"] + FileUtils.batchCreateFile(result, "d:\\数据\\小区\\补充容积绿化" + city + ".csv", headers, "UTF-8") + } + stopWatch.stop() + log.info("完成根据周围500米容积率与绿化率平均值推算, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + } + + static def findResult(lat, lng, radius, districts, region, id) { + def avgMap = [:] + def ret = [id] + if (lat && lng) { + def cubagePercentArr = [] + def virescencePercentArr = [] + def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, radius) + for (def row : districts) { + if (row.lat2 && row.lng2) { + def lat2 = row.lat2 / 10000000000.0; + def lng2 = row.lng2 / 10000000000.0; + if (lat2 >= map.minLat && lat2 <= map.maxLat && lng2 >= map.minLng && lng2 <= map.maxLng) { + def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2) + if (distance <= radius) { + if (row.cubage_percent) { + cubagePercentArr << row.cubage_percent; + } + if (row.virescence_percent) { + virescencePercentArr << row.virescence_percent + } + } + } + } + } + try { + // 计算平均容积率 + if (cubagePercentArr) { + double sum = 0 + double count = 0.0d + for (def m : cubagePercentArr) { + sum = sum + Double.valueOf(m) + count++ + } + if (sum) { + double avgCubagePercent = sum / count + ret << DataHandleUtils.getDoubleRound(2, avgCubagePercent) + } + } else { + ret << "" + } + + // 计算平均绿化率 + if (virescencePercentArr) { + double sum = 0 + double count = 0.0d + for (def m : virescencePercentArr) { + sum += Double.valueOf(m) + count++ + } + if (sum) { + def avgVirescencePercent = sum / (count) + ret << DataHandleUtils.getDoubleRound(2, avgVirescencePercent) + } + } else { + ret << "" + } + log.info("找到可以优化的小区" + region + "周围平均值为:容积率" + ret[1] + ", 绿化率" + ret[2]) + } catch(Exception ex) { + log.error("容积率或绿化率不标准,无法解析") + } + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy new file mode 100644 index 000000000..bdd0d1110 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/geo/BlockRegion.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.geo + +import com.fangjia.script.gj.location.Point +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.xh.InsidePolygon +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "block", "192.168.0.85", 28017) + +collection.find(new BasicDBObject()).forEach({ obj -> + if (obj.type == 'block') { + if (obj.region instanceof String) { + return; + } + BasicDBList reigon = (BasicDBList)obj.region + int count = 0 + List info = new ArrayList<>() + for (String s : reigon) { + info.add(s) + } + if (info.size() > 1) { + boolean find = false + for (String s : info) { + if (find) { + break + } + collection.find(new BasicDBObject("type", "region").append("name", s)).limit(1).forEach({obj2 -> + String border = obj2.border + String points = border.replaceAll(",", "-").replaceAll(";", "-") + List pointList = InsidePolygon.createPoint(points); + com.fangjia.script.looplineinfo.Point point = new com.fangjia.script.looplineinfo.Point(obj.lng, obj.lat) + if (InsidePolygon.aboutZone(point, pointList)) { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", obj2.name))); + find = true; + } + }) + } + } else { + collection.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("region", info.get(0)))); + } + } +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy new file mode 100644 index 000000000..519e5de21 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 商圈经纬度 + */ +class BusinessLatLng { + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(1) + + static void main(String[] args) { + dealPoi(new BasicDBObject("city", "上海").append("region", "浦东新区")) + executorService.shutdown() + } + + static dealPoi(DBObject query) { + collection.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def address = city + obj.region + obj.address; + def location = GaoDeUtils.parserByGaode(city, address); + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + collection.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "已获取经纬度"))) + } else { + collection.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy new file mode 100644 index 000000000..53b0679f0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 国家标准居委会经纬度geohash化 + * + * @author shenjian + * @since 2021/7/22 + */ +class CommitteeGeoHash { + + static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + dealPoi(new BasicDBObject("status_location", 0).append("lng", new BasicDBObject('$ne', null))) + executorService.shutdown() + } + + static dealPoi(DBObject query) { + villagesConn.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + double lng = Double.valueOf(obj.lng) + double lat = Double.valueOf(obj.lat) + def geohash = SeaweedUtils.encodeByGeoHash(lng, lat, 8); + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("geohash", geohash))) + println("已处理小区:" + obj) + } + }) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy new file mode 100644 index 000000000..c1a7d36eb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -0,0 +1,119 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.CosineSimilarAlgorithm +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 国家标准居委会经纬度获取 + * + * @author shenjian + * @since 2021/7/22 + */ +class CommitteeLatLng { + + static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021)) + //dealLocation() + executorService.shutdown() + } + + static dealPoi(DBObject query) { + villagesConn.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def province_name = obj.province_name + def city = obj.city_name + def area = obj.area_name + def name = obj.name + + if (city == "市辖区") { + city = obj.province_name + } else if (city == "省直辖县级行政区划" || city == "自治区直辖县级行政区划") { + city = obj.area_name + area = "" + } else if (city == "县") { + city = obj.province_name + province_name = "" + } + if (obj.name.contains("虚拟生活区") || obj.name.contains("管理区") || obj.name.contains("虚拟社区")) { + name = obj.street_name + } + if (name.contains("社区居民委员会")) { + name = name.replace("居民委员会", "") + } + if (name.contains("村村委会")) { + name = name.replace("村委会", "") + } else if (name.contains("村村民委员会")) { + name = name.replace("村民委员会", "") + } else if (name.contains("村委会")) { + name = name.replace("村委会", "村") + } else if (name.contains("村民委员会")) { + name = name.replace("村民委员会", "村") + } + + String address = province_name + " " + city + " " + area + " " + obj.street_name + " " + name; + //String address = province_name + city + area + name + List amapPoi = GaoDeUtils.searchPoi(city, address, area, "130106|130105|130104") + if (amapPoi && amapPoi.size() > 0) { + def poi = amapPoi[0] + double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, name) + if (similarity <= 0.8) { + amapPoi = GaoDeUtils.searchPoi(city, address, area, "") + } + poi = amapPoi[0] + String[] locations = poi.location.split(",") + def lng = Double.valueOf(locations[0]) + def lat = Double.valueOf(locations[1]) + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度"))) + } else { + amapPoi = GaoDeUtils.searchPoi(city, address, area, "") + if (amapPoi && amapPoi.size() > 0) { + def poi = amapPoi[0] + String[] locations = poi.location.split(",") + def lng = Double.valueOf(locations[0]) + def lat = Double.valueOf(locations[1]) + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度"))) + + } else { + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + } + } + println("已处理小区:" + obj) + } + }) + } + } + + static dealLocation() { + villagesConn.find(new BasicDBObject("status_location", 0)).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + if (obj.lng && obj.lat) { + def lng = Double.valueOf(obj.lng) + def lat = Double.valueOf(obj.lat) + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("location", new BasicDBObject("lat", lat).append("lon", lng)))) + } + println("已处理小区:" + obj) + } + }) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CounterCheckGaoDe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CounterCheckGaoDe.groovy new file mode 100644 index 000000000..44d467755 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CounterCheckGaoDe.groovy @@ -0,0 +1,30 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.ExcelUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils + +/** + * 反查高德小区信息 + */ +class CounterCheckGaoDe { + + static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + List> infos = ExcelUtils.read("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\地图反查信息_高德结果2.xlsx") + for (List info : infos) { + if (info.size() < 6) { + String id = info.get(4) + List amapPois = GaoDeUtils.getAmapById(id) + if (CollectionUtils.isNotEmpty(amapPois)) { + String jsonStr = JSONObject.toJSONString(amapPois) + HttpUtils.postForJson(URL, jsonStr) + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy new file mode 100644 index 000000000..f466ed3be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +final DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + +/** + * 获取省市区编码 + */ +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V9(补全省市区代码).xlsx",{List item-> + + def province = item['C'] + def city = item['D'] + def area = item['E'] + if (StringUtils.isBlank(province) && city == '重庆' && item['K'] == null) { + item['J'] = '5000000' + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(area)) { + villagesConn.find(new BasicDBObject("province_name", "重庆市") + .append("area_name", new BasicDBObject('$regex', '^' + area)), new BasicDBObject() ) + .limit(1).each {obj -> + item['J'] = obj.province_code + item['K'] = obj.area_code + } + } + } else if (StringUtils.isNotBlank(province) && StringUtils.isNotBlank(city) && StringUtils.isNotBlank(area) && item['K'] == null) { + villagesConn.find(new BasicDBObject("province_name", new BasicDBObject('$regex', '^' + province)) + .append("city_name", new BasicDBObject('$regex', '^' + city)) + .append("area_name", new BasicDBObject('$regex', '^' + area)), new BasicDBObject() ) + .limit(1).each {obj -> + item['I'] = obj.province_code + item['J'] = obj.city_code + item['K'] = obj.area_code + } + } + if (item['I'] != null) { + item['I'] = (String.valueOf(item['I']).replace(".0", "") + "000000").substring(0, 6); + } + if (item['J'] != null) { + item['J'] = (String.valueOf(item['J']).replace(".0", "") + "000000").substring(0, 6); + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy new file mode 100644 index 000000000..8125bf1db --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V8.xlsx",{List item-> + + // 乡镇以下级政府及事业单位(居委会) + def types = "130106" + def city = item['D'] + def lng = item['M'] + def lat = item['N'] + if (StringUtils.isBlank(item['G'])) { + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + AmapPoi amapPoi = pois.get(0) + if (amapPoi.getCityname().startsWith(city)) { + item['G'] = amapPoi.getName() + item['S'] = '修改社区' + } + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy new file mode 100644 index 000000000..6b37becf8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy @@ -0,0 +1,17 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import org.apache.commons.lang.StringUtils + +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V8.xlsx",{List item-> + + if (item['L'] != null) { + def location = GeoUtils.gcjToGps84(item['L'], item['K']) + if (location) { + item['N'] = location.lat + item['M'] = location.lng + println(item) + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy new file mode 100644 index 000000000..c23db19ed --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy @@ -0,0 +1,27 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +final DBCollection ZS_REGION_CONN = MongoUtils.getCollectionByName2("fangjia", "region_block", ZS_MONGODB_URL) + +ZS_REGION_CONN.find(new BasicDBObject()).batchSize(1000).each {obj -> + def city = obj.city + def org_name = obj.org_name + + List result = GaoDeUtils.searchPoi(city, org_name, org_name, null) + if (CollectionUtils.isNotEmpty(result)) { + def location = result.get(0).location + def lon = location.split(",")[0] + def lat = location.split(",")[1] + ZS_REGION_CONN.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set',new BasicDBObject("location", new BasicDBObject("lon", lon).append("lat", lat)))) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy new file mode 100644 index 000000000..ef54bf028 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy @@ -0,0 +1,74 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.ExcelUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 获取具体点poi + * + * @author shenjian + * @since 2021/7/26 + */ +class GetPoi { + + static final def URL = "http://192.168.0.216:38080/FangJiaDataService/saveAmapPoi" + static final ExecutorService executorService = Executors.newFixedThreadPool(20) + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0) + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("name", 1).append("city", 1).append("region", 1) + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0).append("city", "重庆") + DbUtils.seaweed.find(query, filter).batchSize(10000).forEach( {seaweedTemp -> + final DBObject seaweed = seaweedTemp; + executorService.submit(new Runnable() { + @Override + void run() { + String city = seaweed.get("city") + String region = seaweed.get("region") + String seaweedName = seaweed.get("name") + List pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302"); + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + atomicLong.incrementAndGet() + } + synchronized (this) { + if (atomicLong.intValue() % 500 == 0) { + println(atomicLong.intValue()) + } + } + } + }) + }) + executorService.shutdown() + } + + + static void getPointInfo() { + List pois = GaoDeUtils.searchNearDtoByPoi("无锡","190301",120.411465,31.864912, 1000, "江苏省江阴市周庄镇兴隆北路"); + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy new file mode 100644 index 000000000..235d779e4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils +import com.fangjia.script.temp.shenjian.util.ExcelUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.io.FileUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class GetPoiFromExcel { + + static final ExecutorService executorService = Executors.newFixedThreadPool(10) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0) + //List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false); + List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Documents\\知识库\\建行\\审核小区"), null, false); + for (File file : fileList) { + List> seaweeds = ExcelUtils.read(file.getPath()) + for (List infoTemp : seaweeds) { + final List info = infoTemp + executorService.submit(new Runnable() { + @Override + void run() { + String city = info.get(0) + String region = info.get(1) + String seaweedName = info.get(2) + List pois = GaoDeUtils.searchPoi(city, seaweedName, region, typecodes); + if (CollectionUtils.isNotEmpty(pois)) { + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); + String jsonStr = JSONObject.toJSONString(detailList) + HttpUtils.postForJson(URL, jsonStr) + atomicLong.incrementAndGet() + } + synchronized (this) { + if (atomicLong.intValue() % 500 == 0) { + println(atomicLong.intValue()) + } + } + } + }) + } + } + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy new file mode 100644 index 000000000..c8481a44a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy @@ -0,0 +1,31 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +final DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + +/** + * 获取国家标准居委街道 + */ +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V9(补全省市区代码).xlsx",{List item-> + + def flag = item['S'] + def area = item['E'] + def name = item['G'] + if (StringUtils.isNotBlank(item['G']) && StringUtils.isNotBlank(area) && String && flag == '修改社区' && item['T'] == null) { + villagesConn.find(new BasicDBObject("province_name", "重庆市") + .append("area_name", area + "区").append("name", name), new BasicDBObject()).limit(1).each { + obj -> + item['F'] = obj.street_name + item['L'] = obj.street_code + item['T'] = "补充街道正确" + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy new file mode 100644 index 000000000..08dc2c38d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy @@ -0,0 +1,74 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.excel.EasyExcel +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.ExcelUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 学校经纬度获取 + * + * @author shenjian + * @since 2021/8/16 + */ +class SchoolLatLng { + + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static String name = "大连市义务教育学校名录test"; + + static void main(String[] args) { + List> schools = ExcelUtils.read(new FileInputStream("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + ".xlsx")); + Map> result = new ConcurrentHashMap<>(); + CountDownLatch latch = new CountDownLatch(schools.size()); + + for (List schoolTemp : schools) { + final List school = schoolTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + def city = "大连" + def list = new ArrayList(); + def address = school.get(1) + def pois = GaoDeUtils.searchPoi(city, address); + if (CollectionUtils.isNotEmpty(pois)) { + for (AmapPoi amapPoi : pois) { + if (address.contains(amapPoi.getName())) { + String[] locations = amapPoi.getLocation().split(",") + list.add(locations[0]) + list.add(locations[1]) + list.add(amapPoi.getId()) + result.put(address, list) + break; + } + } + } + println("已处理:" + address) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + + List> head = new ArrayList<>() + for (List school : schools) { + def address = school.get(1) + if (result.contains(address)) { + school.addAll(result.get(address)) + } + } + EasyExcel.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + "结果.xlsx").head(head).sheet("学校").doWrite(schools) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy new file mode 100644 index 000000000..fc4fecc64 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy @@ -0,0 +1,58 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边居委会获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundCommittee { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + // 乡镇以下级政府及事业单位(居委会) + def types = "130106" + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + static dealPoi(DBObject query, def types) { + query.put("status_committee", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_committee", 0).append("flag_committee", "已获取居委会"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_committee", 2).append("flag_committee", "周边居委会为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..f3c36ebd8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,191 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { + query.put("status", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + public static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } + + public static final Map KEYWORDS_MAP = new HashMap() { + { + put("餐饮", "05"); +// put("地铁", "150500"); +// put("公交", "1507"); +// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); +// put("写字楼", "120201"); +// put("学校", "141200|141201|141202|141203|141204"); +// put("银行", "1601|1603"); +// put("大型超市", "0604"); +// put("商场", "0601"); +// put("餐饮", "0501|0502|0503|0504"); +// put("旅游景点", "110000"); +// put("图书馆", "140500"); +// put("政府机关", "130100|130101|130102|130103|130104|130105|130106|130107"); +// put("丧葬设施", "071900|071901|071902|071903") + + } + }; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy new file mode 100644 index 000000000..20def0a9b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -0,0 +1,145 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.model.Filters +import com.mongodb.client.model.UpdateOneModel +import com.mongodb.client.model.UpdateOptions +import com.mongodb.client.model.WriteModel +import org.apache.commons.collections.CollectionUtils +import org.bson.Document +import org.bson.conversions.Bson +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取单线程方式 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiSingal { + + static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017") + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + getNewSeaweed() +// def types = listAllType() +// //def types = "0601" +// DBObject query = new BasicDBObject() +// query.put("status", 1) +// conn136.find(query).batchSize(30).each {objTemp -> +// final def obj = objTemp +// def city = obj.city +// def lng = obj.lng +// def lat = obj.lat +// List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") +// if (CollectionUtils.isNotEmpty(pois)) { +// String jsonStr = JSONObject.toJSONString(pois) +// HttpUtils.postForJson(URL, jsonStr) +// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) +// } else { +// conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) +// } +// println("已处理小区:" + obj) +// } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("alias", 1) + filter.put("address", 1) + filter.put("addr_alias", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List> writeModelList = new ArrayList<>() + def count = 0 + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + Document updateBson = new Document() + Document update = new Document() + update.put("city", obj.city) + update.put("region", obj.region) + update.put("name", obj.name) + update.put("alias", obj.alias) + update.put("address", obj.address) + update.put("addr_alias", obj.addr_alias) + try { + update.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + update.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + } + update.put("status", 1) + update.put("flag", "未精确获取高德信息") + updateBson.append('$set', update) + + Bson filterBson = Filters.eq("_id", obj._id) + UpdateOptions updateOptions = new UpdateOptions(); + updateOptions.upsert(true) + UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions); + writeModelList.add(updateOneModel) + if (writeModelList.size() == 5000) { + conn136.bulkWrite(writeModelList) + count += 5000 + println(count) + writeModelList.clear() + } + } + if (writeModelList.size() > 0) { + conn136.bulkWrite(writeModelList) + writeModelList.clear() + } + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy new file mode 100644 index 000000000..ea6d9cc67 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedPoiDetail.groovy @@ -0,0 +1,163 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.temp.shenjian.util.CommonDtoUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.model.Filters +import com.mongodb.client.model.UpdateOneModel +import com.mongodb.client.model.UpdateOptions +import com.mongodb.client.model.WriteModel +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.bson.conversions.Bson +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区POI信息精确查找 + * + * @author shenjian + * @since 2022/5/25 + */ +class SeaweedPoiDetail { + + static def conn136 = MongoUtils.getCollectionByName3("fangjia", "seaweed_poi_sync", "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017") + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoiDetail" + // 住宅类 + static final String typecodes = "120000|120100|120200|120201|120202|120203|120300|120301|120302|120303|120304"; + static final ExecutorService executorService = Executors.newFixedThreadPool(20) + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(1000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def region = obj.region + def name = obj.name + def alias = obj.alias + def address = obj.address + def addr_alias = obj.addr_alias + StringJoiner stringJoiner = new StringJoiner("|") + // 主小区名搜索 + stringJoiner.add(name) + // 根据别名搜 + if (alias != null && CollectionUtils.isNotEmpty(alias)) { + for (String str : alias) { + stringJoiner.add(str) + } + } + // 根据路名搜 + if (StringUtils.isNotBlank(address)) { + stringJoiner.add(address) + } + // 根据路别名搜 + if (addr_alias != null && CollectionUtils.isNotEmpty(addr_alias)) { + for (String str : addr_alias) { + stringJoiner.add(str) + } + } + + String keywords = stringJoiner.toString() + List pois = GaoDeUtils.searchPoi(city, keywords, region, typecodes) + if (CollectionUtils.isEmpty(pois)) { + pois = GaoDeUtils.searchPoi(city, name, region, typecodes) + } + if (CollectionUtils.isEmpty(pois)) { + pois = GaoDeUtils.searchPoi(city, address, region, typecodes) + } + + List detailList = CommonDtoUtils.transformListV(AmapPoiDetail.class, pois); +// for (AmapPoi amapPoi : pois) { +// List amapPoiList = GaoDeUtils.getAmapDetailById(amapPoi.getId()); +// if (CollectionUtils.isNotEmpty(amapPoiList)) { +// detailList.addAll(amapPoiList) +// } +// } + if (CollectionUtils.isNotEmpty(detailList)) { + String jsonStr = JSONObject.toJSONString(detailList) + HttpUtils.postForJson(URL, jsonStr) + conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取到"))) + } else { + conn136.updateOne(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "未获取到小区"))) + } + println("已处理小区:" + obj) + } + }) + } + executorService.shutdown() + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("alias", 1) + filter.put("address", 1) + filter.put("addr_alias", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List> writeModelList = new ArrayList<>() + def count = 0 + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + Document updateBson = new Document() + Document update = new Document() + update.put("city", obj.city) + update.put("region", obj.region) + update.put("name", obj.name) + update.put("alias", obj.alias) + update.put("address", obj.address) + update.put("addr_alias", obj.addr_alias) + try { + update.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + update.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + } + update.put("status", 1) + update.put("flag", "未精确获取高德信息") + updateBson.append('$set', update) + + Bson filterBson = Filters.eq("_id", obj._id) + UpdateOptions updateOptions = new UpdateOptions(); + updateOptions.upsert(true) + UpdateOneModel updateOneModel = new UpdateOneModel(filterBson, updateBson, updateOptions); + writeModelList.add(updateOneModel) + if (writeModelList.size() == 5000) { + conn136.bulkWrite(writeModelList) + count += 5000 + println(count) + writeModelList.clear() + } + } + if (writeModelList.size() > 0) { + conn136.bulkWrite(writeModelList) + writeModelList.clear() + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy new file mode 100644 index 000000000..ad1676b7d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy @@ -0,0 +1,181 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 对口小学初中经纬度 + */ +class SeaweedSchoolLatLng { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + static DBCollection school_collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) + + static ExecutorService executorService = Executors.newFixedThreadPool(20) + + static void main(String[] args) { + //latLng() + updateAttribute() + executorService.shutdown() + } + + static void updateAttribute() { + AtomicLong atomicLong = new AtomicLong(0); + collection.find(new BasicDBObject()).batchSize(1000).each {objT -> + final DBObject obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def address = obj.address + boolean update = false + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'district_name': obj.name)) + def quality = '' + + if (primarySchools != null) { + if (schoolObj && schoolObj.attribute) { + quality = schoolObj.attribute + } + for (BasicDBObject dbObject : primarySchools) { + if (dbObject.name == '第六师范学校第二附属小学西校区' || String.valueOf(dbObject.name).startsWith('第六师范附属小学') + || dbObject.name == '上海市第六师范第二附属小学七号桥校区' || dbObject.name == '康城学校康佳校区' || dbObject.name == '三墩学校') { + dbObject.quality = '普通' + update = true + } else if (dbObject.name == '上海市实验学校东校(小学)' || dbObject.name == '浦东新区竹园小学沪东校区') { + dbObject.quality = '市重点' + update = true + } else if (dbObject.name == '浦东新区上南二村小学南校区' || dbObject.name == '浦东新区新时代小学大华校区' + || dbObject.name == '浦东新区明珠小学A区') { + dbObject.quality = '区重点' + update = true + } + else if (quality != '' && schoolObj.category == '小学') { + dbObject.quality = quality + update = true + } else { + update = updateDbObjectQuality(dbObject); + } + } + } + def update2 = false + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + if (dbObject.name == '华东师范大学张江实验中学' || dbObject.name == '10上海市实验学校(初中)(全市招生)' || dbObject.name == '上海市实验学校附属光明学校总校区' + || dbObject.name == '上海市浦东新区进才实验中学南校' || String.valueOf(dbObject.name).startsWith('建平中学') || String.valueOf(dbObject.name).startsWith('建平实验中学') + || dbObject.name == '张江集团中学' || dbObject.name == '洋泾-菊园实验学校' || dbObject.name == '浦东外国语学校东校' || dbObject.name == '浦东新区建平中学南校' || String.valueOf(dbObject.name).startsWith('育人中学') + || dbObject.name == '致远中学' || String.valueOf(dbObject.name).startsWith('进才中学') || String.valueOf(dbObject.name).startsWith('进才实验中学')) { + dbObject.quality = '市重点' + update2 = true + } else if (dbObject.name == '华东师范大学附属东昌中学南校潍坊校区' || dbObject.name == '下沙学校鹤沙路校区' || dbObject.name == '傅雷中学关岳路校区' + || dbObject.name == '傅雷中学瑞和路校区' || dbObject.name == '华东师范大学附属东昌中学南校张江校区' || dbObject.name == '南汇第一中学卫星东路校区' + || dbObject.name == '康城学校(拯安校区)(初中)' || dbObject.name == '建平香梅中学' || dbObject.name == '沪新中学' || dbObject.name == '浦东模范中学' + || dbObject.name == '浦东模范中学东校' || dbObject.name == '浦东模范实验中学' || String.valueOf(dbObject.name).startsWith('澧溪中学') || + dbObject.name == '清流中学' || String.valueOf(dbObject.name).startsWith('王港中学') || dbObject.name == '罗山中学' + || dbObject.name == '金杨中学' || dbObject.name == '香山中学' || dbObject.name == '高行中学') { + dbObject.quality = '区重点' + update2 = true + } else if (quality != '' && schoolObj.category == '初中') { + dbObject.quality = quality + update2 = true + } else { + update2 = updateDbObjectQuality(dbObject) + } + } + } + if (update || update2) { + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) + } + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + atomicLong.incrementAndGet() + synchronized (this) { + if (atomicLong.longValue() % 100 == 0) { + println(atomicLong.longValue()) + } + } + } + } + }) + }; + } + static boolean updateDbObjectQuality(BasicDBObject dbObject) { + if (dbObject.quality == '公办') { + DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'school_name': dbObject.name)) + if (schoolObj && schoolObj.attribute) { + dbObject.quality = schoolObj.attribute + return true + } + } + return false + } + + static void latLng() { + AtomicLong atomicLong = new AtomicLong(0); + collection.find(new BasicDBObject()).batchSize(1000).each {objT -> + final DBObject obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def address = obj.address + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + if (primarySchools != null) { + for (BasicDBObject dbObject : primarySchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + } + } + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + } + } + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + atomicLong.incrementAndGet() + synchronized (this) { + if (atomicLong.longValue() % 100 == 0) { + println(atomicLong.longValue()) + } + } + } + } + }) + }; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy new file mode 100644 index 000000000..cc0cb39dd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.MyStringUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +/** + * @author shenjian* @since 2021/7/7 + */ +class StatisticsPoi { + static final cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京", + "青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原", + "台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳", + "哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江", + "岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳", + "柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照", + "拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + + + static final types = ['地铁', '公交', '医院', '写字楼', '学校', '银行', '大型超市', '商场', '餐饮', '旅游景点', '图书馆'] + + static void main(String[] args) { + def result = [] + + result << ['城市', '数据量', '地铁', '公交', '医院', '写字楼', '学校', '银行', '大型超市', '商场', '餐饮', '旅游景点', '图书馆'] + for (def city in cities) { + def ret = [] + def total + String sql = "select count(*) total from amap_poi where cityname = '" + city + "' and status <> 1 " + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.total + }) + ret << city + ret << total + for (def type in types) { + def typeCode = SeaweedAroundPoi.KEYWORDS_MAP.get(type) + def typeCodeArr = typeCode.split("\\|") + String val = "" + String sqlType = "" + if (typeCodeArr[0].size() <= 4) { + val = MyStringUtils.joinLikeSql(typeCodeArr) + sqlType = "select count(*) total from amap_poi where cityname = '" + city + "' and (" + val + ") and status <> 1 " + } else { + val = MyStringUtils.joinInSql(typeCodeArr) + sqlType = "select count(*) total from amap_poi where cityname = '" + city + "' and typecode in (" + val + ") and status <> 1 " + } + + ElasticUtils.executeSqlQuery(sqlType,{ data-> + total = data.total + }) + ret << total + } + result << ret + println(city) + } + + ExcelUtils.write("F:\\工作\\房产估值模型及数据招标\\交付工作\\POI信息抓取情况总结.xlsx", result, "POI统计") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy new file mode 100644 index 000000000..4764ce68e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy @@ -0,0 +1,69 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.excel.EasyExcel +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.ExcelUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import joptsimple.internal.Strings +import org.apache.commons.collections.CollectionUtils + +/** + * 银联小区经纬度与周边配套 + */ +class YinlianLngLatAndAround { + + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + this.getAround() + } + + public static void getAround() { + List> datas = ExcelUtils.read("D:\\数据\\银联\\测试样本.xlsx") + for (List data : datas) { + if ("城市".equals(data.get(0))) { + continue; + } + if (data.size() == 7) { + String city = data.get(0) + double lng = Double.valueOf(data.get(5)) + double lat = Double.valueOf(data.get(6)) + HttpUtils.postForJson() + List pois = GaoDeUtils.searchNearDtoByPoi(city, SeaweedAroundPoi.listAllType(), lng, lat, 3000, "") + if (CollectionUtils.isNotEmpty(pois)) { +// String jsonStr = JSONObject.toJSONString(pois) +// HttpUtils.postForJson(URL, jsonStr) + List infos = new ArrayList<>() + for (AmapPoi amapPoi : pois) { + infos.add(amapPoi.getName()) + } + data.add(Strings.join(infos, ",")) + } + } + } + def headArr = ["城市","区域","小区名","面积","估值","周边3公里配套", "经度", "纬度"] + def head = ExcelUtils.formatHead(headArr) + EasyExcel.write("D:\\数据\\银联\\测试样本-估值&小区配套3公里(银联测试)结果.xlsx").head(head).sheet("Sheet1").doWrite(datas); + } + + public void getLocation() { + List> datas = ExcelUtils.read("D:\\数据\\银联\\测试样本.xlsx") + for (List data : datas) { + String city = data.get(0) + String region = data.get(1) + String name = data.get(2) + + String address = city + region + name + List lngLat = GaoDeUtils.getLngLat(city, address, region, name, "") + if (CollectionUtils.isNotEmpty(lngLat)) { + data.add(lngLat.get(0)) + data.add(lngLat.get(1)) + } + } + def headArr = ["城市","区域","小区名","面积","估值","周边3公里配套", "经度", "纬度"] + def head = ExcelUtils.formatHead(headArr) + EasyExcel.write("D:\\数据\\银联\\测试样本-估值&小区配套3公里(银联测试)结果.xlsx").head(head).sheet("Sheet1").doWrite(datas); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json new file mode 100644 index 000000000..2cc6544cd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json @@ -0,0 +1,48 @@ +[ + { + "id": "B0FFLCY016", + "parent": [], + "childtype": [], + "name": "杭州馨悦坊公寓", + "type": "住宿服务;住宿服务相关;住宿服务相关", + "typecode": "100000", + "biz_type": "hotel", + "address": "18号大街伊萨卡浩泽园", + "location": "120.383947,30.289931", + "tel": [], + "pname": "浙江省", + "cityname": "杭州市", + "adname": "钱塘区", + "importance": [], + "shopid": [], + "shopinfo": "0", + "poiweight": [], + "distance": "992", + "biz_ext": { + "rating": [], + "cost": [], + "hotel_ordering": "0", + "lowest_price": [], + "star": [], + "open_time": [], + "opentime2": [] + }, + "photos": [ + { + "title": "Logo", + "url": "http://store.is.autonavi.com/showpic/7e9c5bdeeba929eb4742102b7c9bd47c", + "provider": [] + }, + { + "title": "客房", + "url": "http://store.is.autonavi.com/showpic/e4e5771e4de5083877931fda2e360f74", + "provider": [] + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/5401e1e105bcbacf550896c7575ee315", + "provider": [] + } + ] + } +] \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy new file mode 100644 index 000000000..6994cf1dc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy @@ -0,0 +1,95 @@ +package com.fangjia.script.temp.shenjian.qianhai + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.xingyebank.CityUtil +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils + +ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据.xlsx",{List item-> + + if(StringUtils.isBlank(item['B'])){ + return; + } + HttpRequest request = new HttpRequest() + def params = [:] + params["address"] = item['B'] + params["city"] = getCityFormAddress(item['B']) + params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + JSONObject json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + + if((!json || !json.result) && StringUtils.isNotBlank(item['G'])){ + params["address"] = item['G'] + json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + } + + if(json.code == 200 && json.result){ + JSONObject result = json.result + item['C'] = CityUtil.getProvinceByCity(result.city) + item['D'] = result.city + item['E'] = result.district + item['F'] = result.name + if (result.name) { + item['O'] = '住宅地址' + } + item['G'] = result.address + item['H'] = result.lng + item['I'] = result.lat + + // 小区均价查询 + int avgPrice = OpenApiClient.queryDistrictAvgPrice(result.city,result.district,result.name) + if(avgPrice > 0){ + item['M'] = avgPrice + } + params.clear() + params["city"] = result.city + params["district"] = result.district + params["name"] = result.name + params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + // 小区周边一公里均价 + json = request.getJsonContent("http://open.fangjia.com/property/aroundAverage", params, "UTF-8") + if(json.result){ + item['J'] = json.getJSONObject("result").averagePrice + } + } + if (!item['O']) { + item['O'] = '非住宅地址' + } + params.clear() + params["city"] = getCityFormAddress(item['B']) + params["address"] = item['B'] + params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + + // 地址认证接口 + json = request.getJsonContent("http://open.fangjia.com/address/auth", params, "UTF-8") + if(json && json.result){ + JSONObject result = json.result + item['K'] = result.formattedAddress + item['L'] = result.authLevel + } + // 小区详情接口 + params.clear() + params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + params["city"] = getCityFormAddress(item['B']) + params["name"] = item['F'] + params["region"] = item['E'] + + json = request.getJsonContent("http://open.fangjia.com/district/show", params, "UTF-8") + if(json){ + item['N'] = json.buildType + } + + println(item) +}) + +def getCityFormAddress(String addr){ + List cities = CommonUtils.findAllCity() + for(String city : cities){ + if(addr.contains(city)){ + return city + } + } + return null +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy new file mode 100644 index 000000000..f5c6f3504 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.core.util.AddressLatlng +import com.fangjia.script.district.Distance +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +/** + * 处理小区-学校距离过大的数据 + * + * @author shenjian + * @since 2021/4/16 + */ +class DistanceDeal { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) + + // 修复 鑫苑国际城市花园 https://guojichengshihuayuanszxy.fang.com 区域问题 + static void main(String[] args) { + def result = GaoDeApiUtils.geocode("117.363998413086,31.89208984375") + print(result) + + /* def ret = AddressLatlng.getLngLatByAddress("苏州", "苏州工业园区星湾学校(西校区)") + print(ret)*/ + +// DBObject updateField = new BasicDBObject() +// updateField.put("school_lng", "120.68764") +// updateField.put("school_lat", "31.330963") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10483.htm"), new BasicDBObject('$set', updateField), false, true) + + double length = Distance.GetDistance(); + print(length) + +// DBObject updateField = new BasicDBObject() +// updateField.put("school_lng", 120.68764) +// updateField.put("school_lat", 31.330963) +// updateField.put("school_name", "星湾学校西校区") +// updateField.put("school_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") +// updateField.put("gaode_name", "星湾学校西校区") +// updateField.put("gaode_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") +// +// collection.update(new BasicDBObject("match_name", new BasicDBObject('$in', ["星海人家", "新加花园", "中央景城", "加城湖滨公寓", "沁苑小区", "湖畔花园", +// "中茵皇冠国际", "天域三期", "天域", "天域二期"])).append("match_region", "吴中"), new BasicDBObject('$set', updateField), false, true) + +// def location = this.parserByGaode("苏州", "苏州工业园区星湾学校(西校区)") +// print(location) + +// if (location != null) { + //DBObject updateField = new BasicDBObject() +// updateField.put("match_lng", "120.614203") +// updateField.put("match_lat", "31.309876") +// collection.update(new BasicDBObject("url", "https://jingdelu0512.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "120.610651479") +// updateField.put("match_lat", "31.3131018333") +// collection.update(new BasicDBObject("url", "https://wuqufang.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "117.2703018188") +// updateField.put("match_lat", "31.7484989166") +// collection.update(new BasicDBObject("url", "https://binghuqiancheng.fang.com"), new BasicDBObject('$set', updateField), false, true) + +// +// updateField.put("match_lng", "120.610343") +// updateField.put("match_lat", "31.312715") +// collection.update(new BasicDBObject("url", "https://baolinsiqian.fang.com"), new BasicDBObject('$set', updateField), false, true) +// } +// DBObject updateField = new BasicDBObject() +// updateField.put("match_region", "蜀山") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10689.htm"), new BasicDBObject('$set', updateField), false, true) +// } + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy new file mode 100644 index 000000000..95c48476d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy @@ -0,0 +1,101 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang3.StringUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolInfo { + + static void main(String[] args) { + Map> map = new HashMap<>(); + Map> map2 = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String districtName = String.valueOf(data.get(2)) + String key = city + region + if (districtName.contains("期")) { + key += districtName.substring(0, districtName.length() - 2); + if (StringUtils.isNotBlank(data.get(11))) { // 存在对口初中 + map.put(key, data); + } + if (StringUtils.isNotBlank(data.get(6))) { // 存在对口小学 + map2.put(key, data); + } + } else { + key += districtName + if (StringUtils.isNotBlank(data.get(11))) { // 存在对口初中 + map.put(key, data); + } + if (StringUtils.isNotBlank(data.get(6))) { // 存在对口小学 + map2.put(key, data); + } + } + } + print(map.size()) + Long countPrimary = 0; + Long count = 0; + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String districtName = String.valueOf(data.get(2)) + String key = city + region + if (districtName.contains("期")) { + key += districtName.substring(0, districtName.length() - 2) + } else { + key += districtName + } + if (map.get(key) != null) { + List val = map.get(key) + if (StringUtils.isBlank(data.get(11))) { + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + + count++ + println("初中" + data) + } + } + if (map2.get(key) != null) { + List val = map2.get(key) + if (StringUtils.isBlank(data.get(6))) { + data.set(6, val.get(6)) + data.set(7, val.get(7)) + data.set(8, val.get(8)) + data.set(9, val.get(9)) + data.set(10, val.get(10)) + + countPrimary++ + println("小学" + data) + } + } + } + + List> result = new ArrayList<>(); + for (List data : list) { + if (StringUtils.isNotBlank(data.get(6))) { + result.add(data) + } + } + + print("优化数量" + countPrimary + " " + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + result.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", result, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy new file mode 100644 index 000000000..8f4ace4c8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy @@ -0,0 +1,57 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang3.StringUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolWithPrimarySchool { + + static void main(String[] args) { + Map> map = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + // 对口小学 对口中学 + String city = data.get(0) + String region = data.get(1) + if (org.apache.commons.lang.StringUtils.isNotBlank(data.get(7)) && StringUtils.isNotBlank(data.get(11))) { + map.put(city + region + data.get(7), data) + } + } + print(map.size()) + Long count = 0; + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String primarySchool = data.get(7) + + String key = city + region + primarySchool + if (map.get(key) != null && StringUtils.isBlank(data.get(11))) { + List val = map.get(key) + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + count++ + println(data) + } + } + + print("优化数量" + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + list.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", list, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy new file mode 100644 index 000000000..989f77bc6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.ExcelUtils + +/** + * 标准化文件输出 + * + * @author shenjian + * @since 2021/4/16 + */ +class Standard { + + static void main(String[] args) { + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V2.0.xlsx") + def resultOne = [] + def resultTwo = [] + def resultThree = [] + for (List data : list) { + // "320500","610100","340100" + if (data.get(0) == "320500") { + saveData(resultOne, data) + } else if (data.get(0) == "610000") { + data.set(0, 610100) + saveData(resultTwo, data) + } else if (data.get(0) == "340100") { + saveData(resultThree, data) + } + } + def headers = ["城市","区域","小区名称","小区经度","小区纬度", "小区地址","对口小学","小学地址","小学属性","小学批次","小学经度","小学纬度","对口中学","中学地址","中学属性", + "中学批次","中学经度","中学纬度","备注","创建时间","修改时间","来源网站"] + FileUtils.batchCreateFile(resultOne, "F:\\工作\\数据采购\\学校\\bigdata_commschool_320500.dat", headers, "utf-8") + FileUtils.batchCreateFile(resultTwo, "F:\\工作\\数据采购\\学校\\bigdata_commschool_610100.dat", headers, "utf-8") + FileUtils.batchCreateFile(resultThree, "F:\\工作\\数据采购\\学校\\bigdata_commschool_340100.dat", headers, "utf-8") + } + + static void saveData(def result, def data) { + def ret = [] + ret[0] = data.get(0) + ret[1] = data.get(1) + ret[2] = data.get(2) + ret[3] = data.get(3) != null && data.get(3) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(3))) : "" + + ret[4] = data.get(4) != null && data.get(4) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(4))) : "" + ret[5] = data.get(5) + + ret[6] = data.get(6) + ret[7] = data.get(7) + ret[8] = data.get(8) + ret[9] = "" + ret[10] = data.get(9) != null && data.get(9) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(9))) : "" + ret[11] = data.get(10) != null && data.get(10) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(10))) : "" + ret[12] = data.get(11) + ret[13] = data.get(12) + ret[14] = data.get(13) + ret[15] = "" + ret[16] = data.get(14) != null && data.get(14) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(14))) : "" + ret[17] = data.get(15) != null && data.get(15) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(15))) : "" + ret[18] = data.get(16) + ret[19] = data.get(17) + ret[20] = data.get(18) + ret[21] = data.get(19) + + result << ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy new file mode 100644 index 000000000..278446f50 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑结构 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildStructure { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map buildStructureMap = new HashMap() + static { + buildStructureMap.put("钢结构", "3") + buildStructureMap.put("钢、钢筋混凝土结构", "1") + buildStructureMap.put("钢筋混凝土结构", "1") + buildStructureMap.put("钢混", "1") + buildStructureMap.put("钢混结构", "1") + buildStructureMap.put("混合结构", "11") + buildStructureMap.put("钢筋混凝土", "11") + buildStructureMap.put("砖木结构", "10") + buildStructureMap.put("其他结构", "11") + buildStructureMap.put("砖混结构", "2") + buildStructureMap.put("框架结构", "4") + buildStructureMap.put("框架剪力墙结构", "5") + buildStructureMap.put("剪力墙结构", "6") + buildStructureMap.put("筒体结构", "7") + buildStructureMap.put("框架筒体结构", "8") + buildStructureMap.put("筒中筒", "9") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区建筑结构表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("build_structure", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑结构","备注", "创建时间", "更新时间", "来源公司"] + def seaweedCount = [21596, 17053, 5410]; + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑结构\\bigdata_commbuildstru_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑结构表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << this.getBuildStructure(dbo.get("build_structure")) // 建筑结构 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("build_structure"))) { + rows << row + count.incrementAndGet() + } + }) + return rows + }; + + static def getBuildStructure(def houseStructure) { + def ret + if (StringUtils.isNotBlank(houseStructure)) { + ret = buildStructureMap.get(houseStructure) + if (StringUtils.isBlank(ret)) { + ret = "11" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy new file mode 100644 index 000000000..408b87f48 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy @@ -0,0 +1,129 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑类型表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildType { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map buildTypeMap = new HashMap() + static { + buildTypeMap.put("多层", "0104") + buildTypeMap.put("小高层", "0103") + buildTypeMap.put("高层", "0199") + buildTypeMap.put("超高层", "0199") + buildTypeMap.put("花园洋房", "0199") + buildTypeMap.put("老洋房", "0199") + buildTypeMap.put("四合院", "0199") + buildTypeMap.put("独栋", "0105") + buildTypeMap.put("双拼", "0199") + buildTypeMap.put("联排", "0106") + buildTypeMap.put("叠加", "0107") + buildTypeMap.put("老式别墅洋房", "0199") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区建筑类型表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑类型","备注", "创建时间", "更新时间", "来源公司"]; + def seaweedCount = [21596, 17053, 5410]; + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑类型\\bigdata_commbuildtype_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + if (StringUtils.isNotBlank(dbo.get("build_type"))) { + Set set = this.getBuildTypeCode(dbo.get("build_type")) + for (String type : set) { + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << type // 建筑类型 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + rows << row + } + if (set.size() > 0) { + count.incrementAndGet() + } + + } + }) + return rows + } + + + static Set getBuildTypeCode(def buildType) { + Set set = new HashSet<>() + if (StringUtils.isNotBlank(buildType)) { + String[] types = buildType.split(",") + for (String type : types) { + set.add(buildTypeMap.get(type)) + } + if (set.isEmpty()) { + set.add("0199") // 其他 + } + } + return set + } + + static List getBuildTypeCodeSeq(def buildType) { + List list = new ArrayList() + if (StringUtils.isNotBlank(buildType)) { + String[] types = buildType.split(",") + for (String type : types) { + list.add(buildTypeMap.get(type)) + } + if (list.isEmpty()) { + list.add("0199") // 其他 + } + } + return list + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy new file mode 100644 index 000000000..cee15c27c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.text.SimpleDateFormat +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑年代表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildYear { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildYear") + log.info("开始导出小区建筑年代表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("finish_building_date", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑年代","备注", "创建时间", "更新时间", "来源公司"] + + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑年代\\bigdata_commbuildyear_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑年代表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + Integer year = CommonUtils.getBuildYear(dbo.get("finish_building_date")) // 建筑年代 + if (year > 1900) { + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << year + row << "" // 备注 + + Integer createYear = CommonUtils.getBuildYear(DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy")) + if (year > createYear) { + row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date()); + row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date()) + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } + row << "fjw" // 来源公司 + + rows << row + count.incrementAndGet() + } + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy new file mode 100644 index 000000000..e5cbce7af --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy @@ -0,0 +1,87 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.ExcelUtil +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * @author shenjian + * @since 2021-03-13 + */ +class CollectTest { + + static void main(String[] args) { + def cites = ["北京", "重庆", "哈尔滨"]; + for (def city : cites) { + /** + * _id 小区ID + * name 小区名 + * alias 小区曾用名 + */ + def districts = this.findAllCityDistricts(city, "district", + ["city", "_id", "region", "name", "alias"]) + print(districts) + } + } + + static def findAllCityDistricts(def city, def cat, def fields) { + DBObject q = new BasicDBObject(); + q.put("visible", 0) + q.put("cat", cat) + q.put("region", "密云") + if (city) { + q.put("city", city) + } + + DBObject keys = new BasicDBObject(); + if (fields) { + for (def field : fields) { + keys.put(field, 1) + } + } + + DSCursor cursor = CommonDatautils.getDataStoreConnection().find(q, keys).batchSize(4000) + def districtKeys = [] + def rows = []; + rows << ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积", + "小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"] + + for (def dbo : cursor) { + def row = []; + row << null // TODO + row << dbo.get("_id") + row << dbo.get("name") + row << dbo.get("alias") + row << null // TODO + row << dbo.get("street_community") // 街道 TODO 拆分为街道门牌号 + row << dbo.get("street_community") // 门牌号 + rows << dbo.get("address") + rows << dbo.get("all_area") + rows << dbo.get("build_area") + + rows << dbo.get("land_property") + ExcelUtils.write("d:\\数据\\小区表.xlsx", rows, "小区") + + StringBuilder data = new StringBuilder() + Set keySet = dbo.keySet() + keySet.forEach({key -> + if ("alias" == key) { + def value = dbo.get(key) + if (value != null) { + value = value[0] + } + data.append(value) + } else { + data.append(dbo.get(key)) + } + data.append("|@|") + }) + districtKeys << StringUtils.substringBeforeLast(data.toString(), "|@|") + }; + return districtKeys; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy new file mode 100644 index 000000000..87b609daa --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang3.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区公司表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Company { + + private static ExecutorService executorService = Executors.newFixedThreadPool(2); + + // 物业公司 + static final DBCollection PROPERTY_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + // 开发商 + static final DBCollection DEVELOP_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + + static final propertyCompany = new HashMap() + static final developCompany = new HashMap() + + static { + PROPERTY_COMPANY_COLLECTION.find().forEach({company -> + propertyCompany.put(company.get("company"), company.get("_id").toString()) + }) + + DEVELOP_COMPANY_COLLECTION.find().forEach({company -> + developCompany.put(company.get("company"), company.get("_id").toString()) + }) + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Company") + log.info("开始导出小区公司表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_company", 1).append("developer", 1) + .append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID","公司ID","公司类型", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger developCount = new AtomicInteger(0) + AtomicInteger estateCount = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, developCount, estateCount) + println("城市" + city + " " + cityId + "物业数量为" + estateCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((estateCount.get() * 100) / seaweedCount[i])) ) + println("城市" + city + " " + cityId + "开发商数量为" + developCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((developCount.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\公司\\bigdata_commcomp_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区公司表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger developCount, AtomicInteger estateCount) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + def rowCanSave = true + def row1CanSave = true + row << cityId // 城市ID + row << dbo.get("_id").toString() + def propertyCompany = dbo.get("estate_company") + if (StringUtils.isNotBlank(propertyCompany)) { + row << this.getPropertyCompanyId(propertyCompany) // 公司ID + row << "SV" // 公司类型(物业管理公司SV、 房地产开发商DV ) + } else { + rowCanSave = false + } + + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + def row1 = [] + row1 << cityId + row1 << dbo.get("_id").toString() + def developerCompany = dbo.get("developer") + if (StringUtils.isNotBlank(developerCompany)) { + row1 << this.getDevelopCompanyId(developerCompany) // 公司ID + row1 << "DV" // 公司类型(物业管理公司SV、 房地产开发商DV ) + } else { + row1CanSave = false + } + + row1 << "" // 备注 + row1 << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row1 << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row1 << "fjw" // 来源公司 + if (rowCanSave) { + rows << row + estateCount.incrementAndGet() + } + if (row1CanSave) { + rows << row1 + developCount.incrementAndGet() + } + }) + return rows + } + + /** + * 获取开发商公司ID + * @param company 公司名称 + * @return + */ + static def getDevelopCompanyId(String company) { + if (StringUtils.isBlank(company)) { + return "" + } + return developCompany.get(company) + } + + /** + * 获取物业公司ID + * @param company 公司名称 + * @return + */ + static def getPropertyCompanyId(String company) { + if (StringUtils.isBlank(company)) { + return "" + } + return propertyCompany.get(company) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy new file mode 100644 index 000000000..ff7f7d322 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * @author shenjian + * @since 2021/6/4 + */ +class CountSeaweed { + static void main(String[] args) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("cat", "district"); + q.put("visible", 0); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + println(city + " " + DbUtils.seaweed.count(q)) + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy new file mode 100644 index 000000000..614da8edb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区费用表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Fee { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Fee") + log.info("开始导出小区费用表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_company", 1).append("estate_charge", 1) + .append("estate_type2", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID", "公司ID", "费用", "房屋用途","建筑类型", "小区费用类型", "费用说明", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\费用\\bigdata_commfee_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区费用表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def estate_charge = dbo.get("estate_charge") + if (estate_charge && estate_charge.toString().indexOf("~") != -1) { + String[] arr = estate_charge.toString().split("~") + estate_charge = ((Double.valueOf(arr[0]) + Double.valueOf(arr[1])) / 2.0D).toString() + } + if (estate_charge == '-' || estate_charge == '无') { + estate_charge = "" + } + if (StringUtils.isNotBlank(dbo.get("estate_company")) && StringUtils.isNotBlank(estate_charge) + && StringUtils.isNotBlank(dbo.get("estate_type2")) && StringUtils.isNotBlank(dbo.get("build_type"))) { + List buildTypes = BuildType.getBuildTypeCodeSeq(dbo.get("build_type")) + Set savedBuildTypeSet = new HashSet<>() + boolean isFirst = true + for (String buildType : buildTypes) { + if (savedBuildTypeSet.contains(buildType)) { + continue + } + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() // 小区ID + row << Company.getPropertyCompanyId(dbo.get("estate_company")) // 公司ID + + // 处理 db.getCollection('seaweed').find({'estate_charge': '3.3~4.3'}, {'estate_charge': 1}) + // 特殊费用 3.3~4.3 + if (isFirst) { + row << estate_charge // 费用 + isFirst = false + count.incrementAndGet() + } else { + row << "" + break + } + row << HousePurpose.getHousePurposeCode(dbo.get("estate_type2")) // 房屋用途 + row << buildType // 建筑类型 + row << "1" // 小区费用类型 物业费1 电梯费2 垃圾费3 + row << dbo.get("") // 费用说明 + row << dbo.get("") // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + rows << row + + savedBuildTypeSet.add(buildType) + } + } + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy new file mode 100644 index 000000000..ba8dc7c2e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy @@ -0,0 +1,109 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.StringUtil +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区房屋用途表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class HousePurpose { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map housePurposeMap = new HashMap() + static { + housePurposeMap.put("普通住宅", "11") + housePurposeMap.put("别墅", "11") + housePurposeMap.put("商住", "13") + housePurposeMap.put("其他", "99") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区房屋用途表表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_type2", 1).append("household_count", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","房屋用途","户数", "备注", "创建时间", "更新时间", "来源公司"]; + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\房屋用途\\bigdata_commhouseuse_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区房屋用途表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << getHousePurposeCode(dbo.get("estate_type2")) + + if (dbo.get("household_count") != null) { + if (Long.valueOf(dbo.get("household_count")) < 50) { + row << "" // 户数 + } else { + row << dbo.get("household_count") + } + } else { + row << "" // 户数 + } + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + if (dbo.get("m_date")) { + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") + } + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("estate_type2"))) { + rows << row + } + }) + return rows + }; + + static def getHousePurposeCode(def housePurpose) { + def ret + if (StringUtils.isNotBlank(housePurpose)) { + ret = housePurposeMap.get(housePurpose) + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy new file mode 100644 index 000000000..8e77760a9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy @@ -0,0 +1,99 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区房屋类型表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class HouseType { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map houseTypeMap = new HashMap() + static { + houseTypeMap.put("普通住宅", "01") + houseTypeMap.put("商住", "07") + houseTypeMap.put("别墅", "08") + houseTypeMap.put("其他", "99") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("HouseType") + log.info("开始导出小区房屋类型表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_type2", 1).append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID","小区房屋类型","备注", "创建时间", "更新时间", "来源公司"] + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\房屋类型\\bigdata_commhousetype_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区房屋类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << this.getHouseTypeCode(dbo.get("estate_type2")) // 小区房屋类型 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + if (dbo.get("m_date")) { + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") + } + + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("estate_type2"))) { + rows << row + } + }) + return rows + }; + + static def getHouseTypeCode(def houseType) { + def ret + if (StringUtils.isNotBlank(houseType)) { + ret = houseTypeMap.get(houseType) // 房屋类型 + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy new file mode 100644 index 000000000..b1b702fb6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy @@ -0,0 +1,70 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.statistics.seaweed.HouseDetail +import com.fangjia.script.statistics.seaweed.HouseList +import com.fangjia.script.statistics.seaweed.School +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +/** + * @author shenjian* @since 2021/9/2 + */ +class NewHouseFangDi { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + public static Map headerMap(String cook) { + Map headers = new HashMap(); + headers.put("Host", "www.fangdi.com.cn") + headers.put("Connection", "keep-alive") + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36") + headers.put("Content-Type", "application/x-www-form-urlencoded;charset=utf-8") + headers.put("Accept", "application/json, text/javascript, */*; q=0.01") + headers.put("X-Requested-With", "XMLHttpRequest") + headers.put("Origin", "http://www.fangdi.com.cn") + headers.put("Referer", "http://www.fangdi.com.cn/new_house/new_house_list_fast.html?dicPriceID=&dicRegionID=&flatID=&districtID=5391be55e75e5506&check_input2=105C") + headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3") + headers.put("Accept-Encoding", "gzip, deflate") + headers.put("Accept-Language", "zh-CN,zh;q=0.9") + headers.put("Cookie", "www.fangdi.com_http_ic=www.fangdi.com.cn_80_RS; www.fangdi.com.cn=www.fangdi.com.cn_rs3; FSSBBIl1UgzbN7N80S=QS7tHXu4vqldPlDeqJkUCGoyqBmqHzTtyqvrk2ibcRKGVcbPOJ7ERuh5qYgHVD4l; www.fangdi.com_http_ic=www.fangdi.com.cn_80_RS; FSSBBIl1UgzbN7Nenable=true; JSESSIONID1=jw-mUJZsTiFYkXShzn6asMa401YoEZs-0WzhNHrUGId4Jh1_14tj!-338668155; FSSBBIl1UgzbN7N80T=4t2lc885DGECHVNGMwL4w1LcI9v43OwswK7ZOXM5ZbcyVF1UvDPUs59KVOQkq_2hxc0iJrZi3gBiildN1Xfh7xNz81qFz12KwlUJrncPg8mejTeav9tTOgiiPXMZZfJ4rltOCYgOzLLqbj5nG9esEse9UWPYnFdkTWvjpJUTfdyIGg9vB_S9ABAvmir3e89JTbzvfK4.i_IRPFED2hsteZcTiIpNWteOxZMMeDPSnKJei2_Hqhmn7LQMVZanibREAJ8Ah7IxIAwcMtRqAtZLLRpcQMtwnqkEoUPHf435yJVKL20fHui11nhVV4MxTHo9pcoz6L.yhuezCyP5NGTdpV14XV_yBCBhNkSXKWbO.lWAEUQ8yBkMwLLgaYY34sg0Bz_0") + + + return headers; + } + + static void main(String[] args) { + Map headers = headerMap("") + HttpRequest re = new HttpRequest(); + String str = null + try { + Map params = new HashMap(); +// params.put("districtID", "5391be55e75e5506") +// params.put("dicRegionID", "") +// params.put("dicPriceID", "") +// params.put("flatID", "") + params.put("districtID", "5391be55e75e5506") + params.put("dicRegionID", "") + params.put("stateID", "4d29bb4241910ce3") + params.put("houseAreaID", "0") + params.put("dicAvgpriceID", "0") + params.put("dicPositionID", "") + params.put("houseTypeID", "0") + params.put("houseTypeID", "0") + params.put("address", "") + params.put("openingID", "") + params.put("projectName", "") + params.put("currentPage", "1") + str = re.post("http://www.fangdi.com.cn/service/freshHouse/getFastHosueList.action", params, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace() + } + def data = JSONObject.parseObject(str).get("data") + print(data) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy new file mode 100644 index 000000000..269f01ab3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区地理位置表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Position { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Location") + log.info("开始导出小区地理位置表结构表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("lng", 1).append("lat", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","物体类型","物体ID","位置类型", "序号", "经度", "纬度", "海拔", "备注", "创建时间", "更新时间", "来源公司"] + + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\地理位置\\bigdata_position_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区地理位置表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << "5" // 物体类型 小区5 + row << dbo.get("_id").toString() // 物体ID + row << "1" // 位置类型 中心点1 + row << "1" // 序号 + row << DataHandleUtils.getDoubleRound(6, dbo.get("lng") / 10000000000d) // 经度 + row << DataHandleUtils.getDoubleRound(6, dbo.get("lat") / 10000000000d) // 纬度 + row << "" // 海拔 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + rows << row + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy new file mode 100644 index 000000000..0c2f16c13 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy @@ -0,0 +1,288 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.temp.shenjian.dto.SearchAddressDTO +import com.fangjia.script.temp.shenjian.fill.CalcCubageAndVirescence +import com.fangjia.script.temp.shenjian.util.DistrictUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.regex.Pattern + +/** + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Seaweed { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + private static HttpRequest http = new HttpRequest() + private static final String TOKEN = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + private static Map seaweedTypeMap = new HashMap(); + private static Pattern pattern = Pattern.compile("\\s*|\t|\r|\n"); // 取消空格换行等 + private static Map cityDistrictMap = new HashMap(); + + static { + seaweedTypeMap.put("普通住宅", "pa101") + seaweedTypeMap.put("别墅", "pa101") + seaweedTypeMap.put("商住", "pa101") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Seaweed") + log.info("开始导出小区公司表") + //def cites = ["北京", "重庆", "哈尔滨"] + //def seaweedCount = [21596, 17053, 5410]; + def cites = ["哈尔滨"] + def seaweedCount = [5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("name", 1).append("alias", 1).append("street_community", 1) + .append("address", 1).append("all_area", 1).append("build_area", 1).append("region", 1) + .append("estate_type2", 1).append("virescence_percent", 1).append("cubage_percent", 1) + .append("car_site_count", 1).append("c_date", 1).append("m_date", 1) + .append("lng", 1).append("lat", 1) + def headers = ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积", + "小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + final t = i; + def city = cites[t] + def sameRegionDistricts = [:] + def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts) + BasicDBObject cityDistrictInfo = FangJiaAreaMapUtils.convertToOfficial(city) + def cityId = cityDistrictInfo.get("city_code") // 城市ID + final def map = this.findAllCityDistricts(city, "district", cityId, filter, sameRegionDistricts) + executorService.execute(new Runnable() { + @Override + void run() { + def result = [] + AtomicInteger virescenceCount = new AtomicInteger(0) // 绿化率 + AtomicInteger cubageCount = new AtomicInteger(0) // 容积率 + AtomicInteger allAreaCount = new AtomicInteger(0) + AtomicInteger buildAreaCount = new AtomicInteger(0) + for (def key : map.keys()) { + result << map.get(key) + if (map.get(key) && map.get(key)[8] != "") { + allAreaCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[9] != "") { + buildAreaCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[12] != "") { + virescenceCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[13] != "") { + cubageCount.incrementAndGet() + } + } + println("城市" + city + " " + cityId + "绿化率数量为" + virescenceCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((virescenceCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "容积率数量为" + cubageCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((cubageCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "占地面积数量为" + allAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((allAreaCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "建筑面积数量为" + buildAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((buildAreaCount.get() * 100) / seaweedCount[t])) ) + FileUtils.batchCreateFile(result, "d:\\数据\\小区\\bigdata_community_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + fixedThreadPool.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, def sameRegionDistricts) { + def map = new ConcurrentHashMap(); + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + def total = DbUtils.seaweed.find(query, filter).count() + final CountDownLatch latch = new CountDownLatch(total) + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + fixedThreadPool.execute(new Runnable() { + @Override + void run() { + try { + dealSpecialChar(dbo) + def row = [] + + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << dbo.get("name") + if (dbo.get("alias") != null && dbo.get("alias")[0] != null) { + row << pattern.matcher(dbo.get("alias")[0]).replaceAll("") + } else { + row << "" + } + def districtId + if (StringUtils.isBlank(cityDistrictMap.get(dbo.get("region")))) { + districtId = FangJiaAreaMapUtils.convertToOfficial(city, dbo.get("region")).get("district_code") + // 行政区ID + cityDistrictMap.put(dbo.get("region"), districtId) + } else { + districtId = cityDistrictMap.get(dbo.get("region")) + } + row << districtId + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city); + params.put("address", dbo.get("address")); + // 暂时不用解析 +// String s = http.get("http://open.fangjia.com/address/standardize", params, "utf-8"); +// SearchAddressDTO searchAddress = com.alibaba.fastjson.JSONObject.parseObject(s, SearchAddressDTO.class) +// if (searchAddress.code == '200') { +// //row << searchAddress.result.street // 街道 +// row << "" +// row << searchAddress.result.streetNumber // 门牌号 +// } else { +// row << "" +// row << "" +// } + row << "" // 街道 + row << "" // 门牌号 + + row << dbo.get("address") + + if (dbo.get("all_area") && dbo.get("all_area").toString().indexOf("公顷") != -1) { + def val = dbo.get("all_area").toString() + def temp = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000).longValue() + if (temp > 100000) { + row << "" + } else { + row << temp + } + + } else { + if (dbo.get("all_area")) { + if (Double.valueOf(dbo.get("all_area")) > 100000) { + row << "" + } else { + row << dbo.get("all_area") + } + } else { + row << "" + } + } + if (dbo.get("build_area") && Double.valueOf(dbo.get("build_area")) > 1000) { + row << dbo.get("build_area") + } else { + row << "" + } + + row << "pa101" // 小区类型 + row << "" // 是否新盘 + + if (dbo.get("virescence_percent") != null && dbo.get("cubage_percent") != null) { + // 绿化率 + if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) < 1) { + row << Double.valueOf(dbo.get("virescence_percent")) * 100 + } else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) == 1) { + row << Double.valueOf(dbo.get("virescence_percent")) * 10 + } else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) >= 70) { + row << 65 + } else { + row << Double.valueOf(dbo.get("virescence_percent")) + } + + if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 7 && Double.valueOf(dbo.get("cubage_percent")) < 8)) { + row << 6.2 + } else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 8 && Double.valueOf(dbo.get("cubage_percent")) < 9)) { + row << 6.3 + } else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 9 && Double.valueOf(dbo.get("cubage_percent")) < 10)) { + row << 6.5 + } else if (dbo.get("cubage_percent") && Double.valueOf(dbo.get("cubage_percent")) >= 10) { + row << 6.7 + } else { + row << dbo.get("cubage_percent") // 容积率 + } + } else { + def temp = CalcCubageAndVirescence.findResult(dbo.lat / 10000000000.0d, dbo.lng / 10000000000.0d, 500, sameRegionDistricts.get(dbo.region), dbo.name, dbo._id) + if (temp[2] && Double.valueOf(temp[2]) < 1) { + row << Double.valueOf(temp[2]) * 100 + } else if (temp[2] && Double.valueOf(temp[2]) == 1) { + row << Double.valueOf(temp[2]) * 10 + } else if (temp[2] && Double.valueOf(temp[2]) >= 70) { + row << 65 + } else { + row << temp[2] // 绿化率 + } + + if (temp[1] && (Double.valueOf(temp[1]) >= 7 && Double.valueOf(temp[1]) < 8)) { + row << 6.2 + } else if (temp[1] && (Double.valueOf(temp[1]) >= 8 && Double.valueOf(temp[1]) < 9)) { + row << 6.3 + } else if (temp[1] && (Double.valueOf(temp[1]) >= 9 && Double.valueOf(temp[1]) < 10)) { + row << 6.5 + } else if (temp[1] && Double.valueOf(temp[1]) >= 10) { + row << 6.7 + } else { + row << temp[1] // 容积率 + } + } + if (dbo.get("car_site_count") != null) { + if (Long.valueOf(dbo.get("car_site_count")) > 2000 || Long.valueOf(dbo.get("car_site_count")) < 100) { + row << "" + } else { + row << dbo.get("car_site_count") // 车位数量 + } + } else { + row << "" // 车位数量 + } + + row << "" // 介绍 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + map.put(dbo.get("_id").toString(), row) + } catch(Exception e) { + + } finally { + latch.countDown() + } + } + }) + }) + latch.await() + return map + }; + + /** + * 处理特殊字符 + * @param dbo + */ + static def dealSpecialChar(DBObject dbo) { + Set keySet = dbo.keySet() + for (String key : keySet) { + if (String.isInstance(dbo.get(key))) { + dbo.put(key, pattern.matcher(dbo.get(key)).replaceAll("")) + } + } + }; + + static def getSeaweedTypeCode(def value) { + def ret + if (StringUtils.isNotBlank(value)) { + ret = seaweedTypeMap.get(value) + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy new file mode 100644 index 000000000..f8c9beb67 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy @@ -0,0 +1,63 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * 小区别名数量统计 + * + * @author shenjian + * @since 2021/4/15 + */ +class SeaweedAliasExport { + + static void main(String[] args) { + def cities = ["重庆","苏州","昆明","柳州","曲靖","三明","达州","石狮","大理","福安","平潭","迪庆州"] + def result = [] + result << ["城市", "小区数量", "别名数量", "别名覆盖率"] + for (def city : cities) { + def ret = [] + + DBObject query = new BasicDBObject(); + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) + + //def seaweedCount = DBUtils.seaweed.find(query).batchSize(5000).count() + + int seaweedCount = 0; + int aliasCount = 0; + + if (city == "平潭") { + query.put("city", "福州") + query.put("region", city) + } + if (city == "福安") { + query.put("city", "宁德") + query.put("region", city) + } + if (city == "石狮") { + query.put("city", "泉州") + query.put("region", city) + } + DBUtils.seaweed.find(query).batchSize(5000).each {obj -> + if (obj.alias) { + aliasCount++ + } + seaweedCount++ + } + + ret[0] = city + ret[1] = seaweedCount + ret[2] = aliasCount + if (seaweedCount > 0) { + ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(aliasCount) * 100 / Double.valueOf(seaweedCount)) + } + result << ret + } + println(result) + ExcelUtils.write("F:\\数据\\小区别名覆盖率.xlsx", result, "统计") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy new file mode 100644 index 000000000..a0cede622 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy @@ -0,0 +1,51 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.temp.shenjian.util.ExcelUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * 小区geohash导出 + * + * @author shenjian + * @since 2021/5/25 + */ +class SeaweedGeoHashExport { + + static void main(String[] args) { + print(SeaweedUtils.encodeByGeoHash(30.8006287800, 108.3835777247, 8)) + //List> data = com.fangjia.script.util.ExcelUtils.read("D:\\Users\\Domi\\Desktop\\房价网\\项目\\银联\\长宁区融合数据结果20210625.xlsx") +/* for (List str : data) { + String geohash = str.get(0) + str.add(SeaweedUtils.decodeByGeoHash(geohash)) + }*/ + + //com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\银联\\长宁区融合数据结果20210625.xlsx", data, "geohash"); +// DBObject query = new BasicDBObject() +// query.append("cat", "district").append("visible", 0).append("city", "上海").append("region", "长宁") +// +// DBObject filter = new BasicDBObject() +// filter.put("lng", 1) +// filter.put("lat", 1) +// filter.put("name", 1) +// +// def rows = [] +// DbUtils.seaweed.find(query, filter).batchSize(20000).each {objTemp -> +// final def obj = objTemp +// if (obj.lng && obj.lat) { +// def lon = obj.lng +// def lat = obj.lat +// def geohash = SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7) +// rows << [lon, lat, obj.name, geohash] +// } +// } +// def ret = [] +// ret << ["经度", "纬度", "小区名", "小区geohash"] +// ret.addAll(rows) +// com.fangjia.script.util.ExcelUtils.write("D:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy new file mode 100644 index 000000000..334251215 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -0,0 +1,195 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.Constants +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.statistics.seaweed.HouseDetail +import com.fangjia.script.statistics.seaweed.HouseList +import com.fangjia.script.statistics.seaweed.School +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.MongoCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.jdom.Content; + +/** + * 对口小学、初中 + * @author shenjian + * @since 2021/9/2 + */ +class SeaweedSchool { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + public static Map headerMap(String cook) { + Map headers = new HashMap(); + headers.put("Host", "www.mapmapping.com") + headers.put("Connection", "keep-alive") + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("Content-Type", "application/json") + headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3") + headers.put("Referer", "https://servicewechat.com/wx5568ec264e167aeb/9/page-frame.html") + headers.put("Accept-Encoding", "gzip, deflate, br") + + return headers; + } + + static void main(String[] args) { + Map headers = headerMap("") + HttpRequest re = new HttpRequest(); + String str = null + //final String area = "浦东"; + final int perPage = 10 + final int total = 26766; + for (int page = 474; page < total / perPage; page++) { + print("当前页:" + page) + try { + Map params = new HashMap(); + params.put("city", "上海市") + //params.put("area", area) + params.put("page", String.valueOf(page)) + params.put("perPage", String.valueOf(perPage)) + params.put("priceType", "") + params.put("order", "") + params.put("source", "1") + str = re.get("https://mapmapping.com/school-web/house/getHouseInfoList", params, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace() + } + println str + if (StringUtils.isNotEmpty(str)) { + HouseList houseList = JSONObject.parseObject(str, HouseList.class); + if (houseList != null && "000".equals(houseList.getCode())) { + /*Integer total = houseList.getContent().getTotal();*/ + List listDTOS = houseList.getContent().getList(); + if (CollectionUtils.isNotEmpty(listDTOS)) { + for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { + Integer id = listDTO.getId(); + //# 房屋详情 + HouseDetail houseDetail = getHouseDetail(re, id, headers) + // 对口幼儿园 + School nurserySchool = getNurserySchool(re, id, headers) + //# 对口小学 + School primarySchool = getPrimarySchool(re, id, headers) + //# 对口中学 + School middleSchool = getMiddleSchool(re, id, headers) + String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id) + saveData(houseDetail, nurserySchool, primarySchool, middleSchool, url); + } + } + } + } + } + } + + private static void saveData(HouseDetail houseDetail, School nurserySchool, School primarySchool, School middleSchool, def url) { + DBObject dbObject = new BasicDBObject() + dbObject.put("url", url) + dbObject.put("root", "沪上学区房") + dbObject.put("c_date", new Date()) + dbObject.put("city", "上海") + dbObject.put("region", houseDetail.getContent().getArea()) + dbObject.put("name", houseDetail.getContent().getName()) + dbObject.put("address", houseDetail.getContent().getAddress()) + dbObject.put("lng", houseDetail.getContent().getLng()) + dbObject.put("lat", houseDetail.getContent().getLat()) + + List nurserySchools = nurserySchool.getContent().getList(); + if (CollectionUtils.isNotEmpty(nurserySchools)) { + List schools = buildSchool(nurserySchools) + dbObject.put("nurserySchool", schools) + } + + List primarySchools = primarySchool.getContent().getList();; + if (CollectionUtils.isNotEmpty(primarySchools)) { + List schools = buildSchool(primarySchools) + dbObject.put("primarySchool", schools) + } + + List middleSchools = middleSchool.getContent().getList();; + if (CollectionUtils.isNotEmpty(middleSchools)) { + List schools = buildSchool(middleSchools) + dbObject.put("middleSchool", schools) + } + + collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false) + } + + private static List buildSchool(List currSchools) { + List schools = new ArrayList<>() + for (School.ContentDTO.ListDTO contentDTO : currSchools) { + DBObject schoolObject = new BasicDBObject(); + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(contentDTO.getTags())) { + if (contentDTO.getTags().size() >= 3) { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", contentDTO.getTags().get(1)) + schoolObject.put("batch", contentDTO.getTags().get(2)) + } else if (contentDTO.getTags().size() == 2) { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", contentDTO.getTags().get(1)) + schoolObject.put("batch", '') + } else { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", '') + schoolObject.put("batch", '') + } + } + schoolObject.put("area", contentDTO.getArea()) + schoolObject.put("town", contentDTO.getTown()) + schoolObject.put("distance", contentDTO.getDistance()) + schoolObject.put("address", contentDTO.getAddress()) + schoolObject.put("name", contentDTO.getName()) + schools.add(schoolObject) + } + return schools + } + + + + private static HouseDetail getHouseDetail(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getHouseInfoDetail", params, headers, "UTF-8"); + return JSONObject.parseObject(str, HouseDetail.class) + } + + private static School getNurserySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "1") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + + private static School getPrimarySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "2") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + + private static School getMiddleSchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "3") + params.put("page", "1") + params.put("perPage", "4") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy new file mode 100644 index 000000000..2949d3803 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy @@ -0,0 +1,19 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +/** + * @author shenjian + * @since 2021/6/2 + */ +class CityUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static listCitySet() { + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + return citySet + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy new file mode 100644 index 000000000..89dda4684 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CommonDtoUtils.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.temp.shenjian.util; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import com.alibaba.fastjson.JSONObject; + +/** + * 通用实体与DTO互相转换工具类 + * + * @author shenjian + * @date 2021/06/16 + */ +public class CommonDtoUtils { + + public static T transformV(Class targetClass, Object source) { + if (source == null) { + return null; + } else { + try { + String jsonSource = JSON.toJSONString(source); + return JSONObject.parseObject(jsonSource, targetClass); + } catch (Exception var3) { + throw var3; + } + } + } + + public static List transformListV(Class targetClass, List listSource) { + if (listSource == null) { + return null; + } else { + try { + String jsonSource = JSON.toJSONString(listSource); + return JSONArray.parseArray(jsonSource, targetClass); + } catch (Exception var3) { + throw var3; + } + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy new file mode 100644 index 000000000..03d95a32c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * 区域工具类 + */ +class DistrictUtils { + + static main(args) { + } + + static def getAllDistricts(def cat, def city, def sameRegionDistricts){ + DBObject query = new BasicDBObject(); + query.append("cat", cat).append("visible", 0).append("city", city) + + DBObject keys = new BasicDBObject().append("city", 1).append("region", 1).append("cubage_percent", 1) + .append("virescence_percent", 1).append("lat2", 1).append("lng2", 1) + .append("finish_building_date", 1).append("name", 1).append("_id", 1); + + DSCursor cursor = DbUtils.seaweed.find(query, keys).batchSize(10000) + Iterator iters = cursor.iterator() + def result = [] + while(iters.hasNext()) { + DBObject obj = iters.next() + + def map = [:]; + map.city = obj.city + map.cubage_percent = obj.cubage_percent; + map.virescence_percent = obj.virescence_percent; + map.lat2 = obj.lat2 + map.lng2 = obj.lng2 + map.region = obj.region + map.finish_building_date = obj.finish_building_date + map.name = obj.name + map._id = obj._id + result << map; + if (sameRegionDistricts.get(obj.region)) { + def temp = sameRegionDistricts.get(obj.region) + temp << map + sameRegionDistricts.put(obj.region, temp) + } else { + def temp = [] + temp << map + sameRegionDistricts.put(obj.region, temp) + } + } + return result; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy new file mode 100644 index 000000000..6f01207ca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -0,0 +1,113 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.excel.EasyExcel +import com.alibaba.excel.EasyExcelFactory +import com.alibaba.excel.ExcelReader +import com.alibaba.excel.context.AnalysisContext +import com.alibaba.excel.event.AnalysisEventListener +import com.alibaba.excel.read.metadata.ReadSheet + +import java.util.concurrent.ConcurrentHashMap + +/** + * excel工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class ExcelUtils { + + static List> formatHead(def headArr) { + List> head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + return head; + } + + static List> read(String filePath) { + SimpleExcelListener listener = new SimpleExcelListener(filePath); + ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(1).build(); + excelReader.readAll() + List> dataList = listener.getDataList(); + excelReader.finish() + return dataList; + } + + static List> readSheet(String fileName, String[] sheetNames) { + Map>> result = new ConcurrentHashMap<>(); + ExcelReader excelReader = EasyExcel.read(fileName).build(); + + // 每个sheet分别创建监听器 + SimpleExcelListener listener0 = new SimpleExcelListener(); + SimpleExcelListener listener1 = new SimpleExcelListener(); + + ReadSheet readSheet0 = EasyExcel.readSheet(0).registerReadListener(listener0).build(); + + ReadSheet readSheet1 = EasyExcel.readSheet(1).registerReadListener(listener1).build(); + + // 这里注意 一定要把sheet1 sheet2 一起传进去,不然有个问题就是03版的excel 会读取多次,浪费性能 + // 可以传多个ReadSheet对象,读多个sheet + excelReader.read(readSheet0, readSheet1); + + // 这里千万别忘记关闭,读的时候会创建临时文件,到时磁盘会崩的 + excelReader.finish(); + + ArrayList listenerList = new ArrayList<>(); + listenerList.add(listener0); + listenerList.add(listener1); + + for (SimpleExcelListener listener : listenerList) { + result.put(listener.getSheetName(), listener.getDataList()) + } + return result + } + + public static List> read(InputStream inputStream) { + SimpleExcelListener listener = new SimpleExcelListener(); + ExcelReader excelReader = EasyExcelFactory.read(inputStream, null, listener) + .headRowNumber(1).build(); + excelReader.readAll(); + List> dataList = listener.getDataList(); + excelReader.finish(); + return dataList; + } + + public static class SimpleExcelListener extends AnalysisEventListener { + private List> dataList = new ArrayList<>(); + private String sheetName; + + public SimpleExcelListener() { + + } + + public SimpleExcelListener(String sheetName) { + this.sheetName = sheetName + } + + @Override + public void invoke(Object o, AnalysisContext analysisContext) { + Map map = (Map) o; + dataList.add(new ArrayList<>(map.values())); + } + + @Override + public void doAfterAllAnalysed(AnalysisContext analysisContext) { + + } + + List> getDataList() { + return dataList; + } + + void setDataList(List> dataList) { + this.dataList = dataList; + } + + public String getSheetName() { + return sheetName + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..6b880c5ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,458 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.CosineSimilarAlgorithm +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.statistics.poi.AmapPoiDetail +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import org.apache.commons.lang.StringUtils +import org.openqa.selenium.json.Json + +import java.util.concurrent.atomic.AtomicInteger + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + // 新获取key + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418"] + def static AtomicInteger keyUsed = new AtomicInteger(0) + + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + //System.exit(1) + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找最可能的地址经纬度 + * + * @param city + * @param address + * @param region + * @param typecodes,如130106|130105|130104 + * @return + */ + static List getLngLat(def city, def address, def region, def name, def typecodes) { + List amapPoi = GaoDeUtils.searchPoi(city, address, region, typecodes) + if (amapPoi && amapPoi.size() > 0) { + def poi = amapPoi[0] + double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, name) + if (similarity < 0.7) { + amapPoi = GaoDeUtils.searchPoi(city, address, region, "") + } + poi = amapPoi[0] + String[] locations = poi.location.split(",") + def lng = Double.valueOf(locations[0]) + def lat = Double.valueOf(locations[1]) + return Arrays.asList(lng, lat) + } else { + amapPoi = GaoDeUtils.searchPoi(city, address, region, "") + if (amapPoi && amapPoi.size() > 0) { + def poi = amapPoi[0] + String[] locations = poi.location.split(",") + def lng = Double.valueOf(locations[0]) + def lat = Double.valueOf(locations[1]) + return Arrays.asList(lng, lat) + } + } + return Collections.emptyList(); + } + + /** + * 获取详情 + */ + static List getAmapById(id){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + params.id = id + params.output = 'json' + JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + } + if (json && json.info && json.info == "OK") { + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + return ret + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + //System.exit(1) + Thread.sleep(60000) + } + return Collections.emptyList() + } + + /** + * 获取详情 + */ + static List getAmapDetailById(id){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + params.id = id + params.output = 'json' + JSONObject json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("https://restapi.amap.com/v3/place/detail", params); + } + if (json && json.info && json.info == "OK") { + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + return ret + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + //System.exit(1) + Thread.sleep(60000) + } + return Collections.emptyList() + } + + /** + * 查找关键字经纬度 + */ + static List searchPoi(String city, String keywords, String region, String typecodes){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + params.keywords = keywords + params.city = city + params.region = region + if (StringUtils.isNotBlank(typecodes)) { + params.types = typecodes + } + params.put("extensions", "all") + params.put("citylimit","true") + params.put("offset","20") + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && (json.infocode == "10004" || json.infocode == "10021" || json.infocode == "10017")) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else if (json && json.infocode && (json.infocode == "20000")) { + // 无效参数 + break; + } + } + return result + } + + /** + * 获取经纬度之间距离 + */ + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + /** + * 经纬度转地址
    + * 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位 + * @author yinjihuan + * @date 2016年9月9日 + * @param location + * @return + */ + static def getGaoDeAddress(location) { + Map params = new HashMap() + params.put("key", GAO_DE_AK[keyUsed.intValue()]) + params.put("location", location) + String content + try { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } + net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) + Map infoMap = DataHandleUtils.JsonToMap(json); + if (infoMap && infoMap.info == "OK") { + def regeocode = infoMap.regeocode + if (regeocode) { + return regeocode.formatted_address; + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return null + } + + /** + * 获取地址经纬度 + */ + static def parserByGaode(def city, def address) { + Map params = new HashMap() + params.put("key", GAO_DE_AK[keyUsed.intValue()]) + params.put("city", city) + params.put("address", address) + String content + try { + content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } + net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) + if (json && json.info && json.info == "OK") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy new file mode 100644 index 000000000..93174d945 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSON +import org.msgpack.MessagePack +import org.msgpack.type.Value; + +/** + * msg解析工具类 + */ +public class MessageUtils { + + public static String unpack(byte[] content) { + MessagePack messagePack = new MessagePack(); + try { + Value value = messagePack.read(content); + return value.toString(); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + + public static byte[] pack(String value){ + MessagePack messagePack = new MessagePack(); + try { + byte[] data = messagePack.write(value) + return data; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + public static byte[] pack(JSON json){ + MessagePack messagePack = new MessagePack(); + try { + byte[] data = messagePack.write(json) + return data; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy new file mode 100644 index 000000000..36eb6c935 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.temp.shenjian.util + +/** + * @author shenjian* @since 2021/7/7 + */ +public class MyStringUtils { + + public static joinSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append("'").append(str[i]).append("'") + if (i != str.size() - 1) { + sb.append(",") + } + } + return sb.toString() + } + + public static joinInSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append("'").append(str[i]).append("'") + if (i != str.size() - 1) { + sb.append(",") + } + } + return sb.toString() + } + + public static joinLikeSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append(" typecode like '").append(str[i]).append("%' ") + if (i != str.size() - 1) { + sb.append(" or ") + } + } + return sb.toString() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..c17285a78 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,209 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (val == 1) { + return val * 10 + } + if (val >= 70) { + return 65 + } + return val + } + return "" + } + + /** + * 标准化容积率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (val >= 10) { + return 6.7 + } + return val + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) + if (val > 2500 || val < 100) { + return "" + } + return val; + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Integer.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count.toInteger() + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return DataHandleUtils.getDoubleRound(6, Geohash.decodeLatitude(geohash)) + "," + + DataHandleUtils.getDoubleRound(6, Geohash.decodeLongitude(geohash)) + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy index a284b5c6f..d283bbf5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy @@ -98,9 +98,9 @@ class CrawalUtil { } - static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null) { + static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null, def isJson=false) { def flag = true; - Document document = null; + def document = null; def count = 0; while (flag) { def content = null; @@ -121,7 +121,11 @@ class CrawalUtil { } } if (content) { - document = Jsoup.parse(content); + if(!isJson){ + document = Jsoup.parse(content); + }else{ + document = content + } } if (!content || (content && errorMsg && content.contains(errorMsg))) { count++; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 89b54feea..9a9fd90ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -364,7 +364,7 @@ class DataStandardBatch extends BaseBatch{ def standardDistrict(def data, def avg_price){ //名称匹配,名称别名匹配,地址别名匹配 //def district = getMatchName(data.city, data.region, data.name, data.address,"district") - data.name = data.name.replace("(","").replace(")","") + data.name = data.name.replace("(","").replace(")","").replace("数据由万科物业提供","").trim() def district = null; def type = 0; if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){ @@ -514,6 +514,7 @@ class DataStandardBatch extends BaseBatch{ def filterSpacialChar(def s){ def pattern = ~/[()!;\·]\[【】()]/ def matcher = pattern.matcher(s) + def count = matcher.getCount() if(count && count>0){ return matcher.replaceAll('_') @@ -752,7 +753,7 @@ class DataStandardBatch extends BaseBatch{ if(regionMap){ regionMap.each{k,v-> def arrays = k.split("###") - if(arrays[0]){ + if(arrays[0] && arrays.size()==2){ def py2 = com.fangjia.core.util.DataUtil.hanyuQuanpin(arrays[0], " ") if(py1 == py2){ districts << [city:city, region:region, name:arrays[0],address:arrays[1]] @@ -782,7 +783,6 @@ class DataStandardBatch extends BaseBatch{ def findAllNoMatchData(){ DBObject qdata = new BasicDBObject(); qdata.put("s_date", SHORT_DATE) - qdata.put("city", "福州") qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false)); def ret = [] try{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index fb2c38235..de2db639a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -41,10 +41,10 @@ class DataStat { def run(){ def source = "58同城"// - def citys = findCrawlCity(source, 202107); + def citys = findCrawlCity(source, 202202); for(def city : citys){ - def count1 = findCount(source, city, 202107); - def count2 = findCount(source, city, 202110); + def count1 = findCount(source, city, 202203); + def count2 = findCount(source, city, 202204); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index d7e1021de..ed1b90eba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import net.sf.json.JSONObject import org.jsoup.nodes.Element import org.jsoup.select.Elements @@ -35,43 +36,47 @@ class Esf58Crawl extends BaseBatch{ def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def static final distMap = [:] + //def static useDistKeys = ["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","不","了","这","中","大","有","和","人","是","在","一","的","为","上","个","国","我","以","要","他","时","来","用","们","生","到","作","地","于","出","就","分","对","成","会","可","主","发","年","动","同","工","也","能","下","过","子","说","产","种","面","而","方","后","多","定","行","学","法","所","民","得","经","十","三","之","进","着","等","部","度","家","电","力","里","如","水","化","高","自","二","理","起","小","物","现","实","加","量","都","两","体","制","机","当","使","点","从","业","本","去","把","性","好","应","开","它","合","还","因","由","其","些","然","前","外","天","政","四","日","那","社","义","事","平","形","相","全","表","间","样","与","关","各","重","新","线","内","数","正","心","反","你","明","看","原","又","么","利","比","或","但","质","气","第","向","道","命","此","变","条","只","没","结","解","问","意","建","月","公","无","系","军","很","情","者","最","立","代","想","已","通","并","提","直","题","党","程","展","五","果","料","象","员","革","位","入","常","文","总","次","品","式","活","设","及","管","特","件","长","求","老","头","基","资","边","流","路","级","少","图","山","统","接","知","较","长","将","组","见","计","别","她","手","角","期","根","论","运","农","指","几","九","区","强","放","决","西","被","干","做","必","战","先","回","则","任","取","据","处","队","南","给","色","光","门","即","保","治","北","造","百","规","热","领","七","海","地","口","东","导","器","压","志","世","金","增","争","济","阶","油","思","术","极","交","受","联","什","认","六共","权","收","证","改","清","已","美","再","采","转","更","单","风","切","打","白","教","速","花","带","安","场","身","车","例","真","务","具","万","每","目","至","达","走","积","示","议","声","报","斗","完","类","八","离","华","名","确","才","科","张","信","马","节","话","米","整","空","元","况","今","集","温","传","土","许","步","群","广","石","记","需","段","研","界","拉","林","律","叫","且","究","观","越","织","装","影","算","低","持","音","众","书","布","复","容","儿","须","际","商","非","验","连","断","深","难","近","矿","千","周","委","素","技","备","半","办","青","省","列","习","响","约","支","般","史","感","劳","便","团","往","酸","历","市","克","何","除","消","构","府","称","太","准","精","值","号","率","族","维","划","选","标","写","存","候","毛","亲","快","效","斯","院","查","江","型","眼","王","按","格","养","易","置","派","层","片","始","却","专","状","育","厂","京","识","适","属","圆","包","火","住","调","满","县","局","照","参","红","细","引","听","该","铁","价","严","首","底","液","官","德","调","随","病","苏","失","尔","死","讲","配","女","黄","推","显","谈","罪","神","艺","呢","席","含","企","望","密","批","营","项","防","举","球","英","氧","势","告","李","台","落","木","帮","轮","破","亚","师","围","注","远","字","材","排","供","河","态","封","另","施","减","树","溶","怎","止","案","言","士","均","武","固","叶","鱼","波","视","仅","费","紧","爱","左","章","早","朝","害","续","轻","服","试","食","充","兵","源","判","护","司","足","某","练","差","致","板","田","降","黑","犯","负","击","范","继","兴","似","余","坚","曲","输","修","的","故","城","夫","够","送","笔","船","占","右","财","吃","富","春","职","觉","汉","画","功","巴","跟","虽","杂","飞","检","吸","助","升","阳","互","初","创","抗","考","投","坏","策","古","径","换","未","跑","留","钢","曾","端","责","站","简","述","钱","副","尽","帝","射","草","冲","承","独","令","限","阿","宣","环","双","请","超","微","让","控","州","良","轴","找","否","纪","益","依","优","顶","础","载","倒","房","突","坐","粉","敌","略","客","袁","冷","胜","绝","析","块","剂","测","丝","协","重","诉","念","陈","仍","罗","盐","友","洋","错","苦","夜","刑","移","频","逐","靠","混","母","短","皮","终","聚","汽","村","云","哪","既","距","卫","停","烈","央","察","烧","行","迅","境","若","印","洲","刻","括","激","孔","搞","甚","室","待","核","校","散","侵","吧","甲","游","久","菜","味","旧","模","湖","货","损","预","阻","毫","普","稳","乙","妈","植","息","扩","银","语","挥","酒","守","拿","序","纸","医","缺","雨","吗","针","刘","啊","急","唱","误","训","愿","审","附","获","茶","鲜","粮","斤","孩","脱","硫","肥","善","龙","演","父","渐","血","欢","械","掌","歌","沙","著","刚","攻","谓","盾","讨","晚","粒","乱","燃","矛","乎","杀","药","宁","鲁","贵","钟","煤","读","班","伯","香","介","迫","句","丰","培","握","兰","担","弦","蛋","沉","假","穿","执","答","乐","谁","顺","烟","缩","征","脸","喜","松","脚","困","异","免","背","星","福","买","染","井","概","慢","怕","磁","倍","祖","皇","促","静","补","评","翻","肉","践","尼","衣","宽","扬","棉","希","伤","操","垂","秋","宜","氢","套","笔","督","振","架","亮","末","宪","庆","编","牛","触","映","雷","销","诗","座","居","抓","裂","胞","呼","娘","景","威","绿","晶","厚","盟","衡","鸡","孙","延","危","胶","还","屋","乡","临","陆","顾","掉","呀","灯","岁","措","束","耐","剧","玉","赵","跳","哥","季","课","凯","胡","额","款","绍","卷","齐","伟","蒸","殖","永","宗","苗","川","炉","岩","弱","零","杨","奏","沿","露","杆","探","滑","镇","饭","浓","航","怀","赶","库","夺","伊","灵","税","了","途","灭","赛","归","召","鼓","播","盘","裁","险","康","唯","录","菌","纯","借","糖","盖","横","符","私","努","堂","域","枪","润","幅","哈","竟","熟","虫","泽","脑","壤","碳","欧","遍","侧","寨","敢","彻","虑","斜","薄","庭","都","纳","弹","饲","伸","折","麦","湿","暗","荷","瓦","塞","床","筑","恶","户","访","塔","奇","透","梁","刀","旋","迹","卡","氯","遇","份","毒","泥","退","洗","摆","灰","彩","卖","耗","夏","择","忙","铜","献","硬","予","繁","圈","雪","函","亦","抽","篇","阵","阴","丁","尺","追","堆","雄","迎","泛","爸","楼","避","谋","吨","野","猪","旗","累","偏","典","馆","索","秦","脂","潮","爷","豆","忽","托","惊","塑","遗","愈","朱","替","纤","粗","倾","尚","痛","楚","谢","奋","购","磨","君","池","旁","碎","骨","监","捕","弟","暴","割","贯","殊","释","词","亡","壁","顿","宝","午","尘","闻","揭","炮","残","冬","桥","妇","警","综","招","吴","付","浮","遭","徐","您","摇","谷","赞","箱","隔","订","男","吹","乐","园","纷","唐","败","宋","玻","巨","耕","坦","荣","闭","湾","键","凡","驻","锅","救","恩","剥","凝","碱","齿","截","炼","麻","纺","禁","废","盛","版","缓","净","睛","昌","婚","涉","筒","嘴","插","岸","朗","庄","街","藏","姑","贸","腐","奴","啦","惯","乘","伙","恢","匀","纱","扎","辩","耳","彪","臣","亿","璃","抵","脉","秀","萨","俄","网","舞","店","喷","纵","寸","汗","挂","洪","着","贺","闪","柬","爆","烯","津","稻","墙","软","勇","像","滚","厘","蒙","芳","肯","坡","柱","荡","腿","仪","旅","尾","轧","冰","贡","登","黎","削","钻","勒","逃","障","氨","郭","峰","币","港","伏","轨","亩","毕","擦","莫","刺","浪","秘","援","株","健","售","股","岛","甘","泡","睡","童","铸","汤","阀","休","汇","舍","牧","绕","炸","哲","磷","绩","朋","淡","尖","启","陷","柴","呈","徒","颜","泪","稍","忘","泵","蓝","拖","洞","授","镜","辛","壮","锋","贫","虚","弯","摩","泰","幼","廷","尊","窗","纲","弄","隶","疑","氏","宫","姐","震","瑞","怪","尤","琴","循","描","膜","违","夹","腰","缘","珠","穷","森","枝","竹","沟","催","绳","忆","邦","剩","幸","浆","栏","拥","牙","贮","礼","滤","钠","纹","弹","罢","拍","咱","喊","袖","埃","勤","罚","焦","潜","伍","墨","欲","缝","姓","刊","饱","仿","奖","铝","鬼","丽","跨","默","挖","链","扫","喝","袋","炭","污","幕","诸","弧","励","梅","奶","洁","灾","舟","鉴","苯","讼","抱","毁","率","懂","寒","智","埔","寄","届","跃","渡","挑","丹","艰","贝","碰","拔","爹","戴","码","梦","芽","熔","赤","渔","哭","敬","颗","奔","藏","铅","熟","仲","虎","稀","妹","乏","珍","申","桌","遵","允","隆","螺","仓","魏","锐","晓","氮","兼","隐","碍","赫","拨","忠","肃","缸","牵","抢","博","巧","壳","兄","杜","讯","诚","碧","祥","柯","页","巡","矩","悲","灌","龄","伦","票","寻","桂","铺","圣","恐","恰","郑","趣","抬","荒","腾","贴","柔","滴","猛","阔","辆","妻","填","撤","储","签","闹","扰","紫","砂","递","戏","吊","陶","伐","喂","疗","瓶","婆","抚","臂","摸","忍","虾","蜡","邻","胸","巩","挤","偶","弃","槽","劲","乳","邓","吉","仁","烂","砖","租","乌","舰","伴","瓜","浅","丙","暂","燥","橡","柳","迷","暖","牌","纤","秧","胆","详","簧","踏","瓷","谱","呆","宾","糊","洛","辉","愤","竞","隙","怒","粘","乃","绪","肩","籍","敏","涂","熙","皆","侦","悬","掘","享","纠","醒","狂","锁","淀","恨","牲","霸","爬","赏","逆","玩","陵","祝","秒","浙","貌","役","彼","悉","鸭","着","趋","凤","晨","畜","辈","秩","卵","署","梯","炎","滩","棋","驱","筛","峡","冒","啥","寿","译","浸","泉","帽","迟","硅","疆","贷","漏","稿","冠","嫩","胁","芯","牢","叛","蚀","奥","鸣","岭","羊","凭","串","塘","绘","酵","融","盆","锡","庙","筹","冻","辅","摄","袭","筋","拒","僚","旱","钾","鸟","漆","沈","眉","疏","添","棒","穗","硝","韩","逼","扭","侨","凉","挺","碗","栽","炒","杯","患","馏","劝","豪","辽","勃","鸿","旦","吏","拜","狗","埋","辊","掩","饮","搬","骂","辞","勾","扣","估","蒋","绒","雾","丈","朵","姆","拟","宇","辑","陕","雕","偿","蓄","崇","剪","倡","厅","咬","驶","薯","刷","斥","番","赋","奉","佛","浇","漫","曼","扇","钙","桃","扶","仔","返","俗","亏","腔","鞋","棱","覆","框","悄","叔","撞","骗","勘","旺","沸","孤","粘","吐","孟","渠","屈","疾","妙","惜","仰","狠","胀","谐","抛","霉","桑","岗","嘛","衰","盗","渗","脏","赖","涌","甜","曹","阅","肌","哩","厉","烃","纬","毅","昨","伪","症","煮","叹","钉","搭","茎","笼","酷","偷","弓","锥","恒","杰","坑","鼻","翼","纶","叙","狱","逮","罐","络","棚","抑","膨","蔬","寺","骤","穆","冶","枯","册","尸","凸","绅","坯","牺","焰","轰","欣","晋","瘦","御","锭","锦","丧","旬","锻","垄","搜","佛","扑","邀","亭","酯","迈","舒","脆","酶","闲","忧","酚","顽","羽","涨","卸","仗","陪","薄","辟","惩","杭","姚","肚","捉","飘","漂","昆","欺","吾","郎","烷","汁","呵","饰","萧","雅","邮","迁","燕","撒","姻","赴","宴","烦","削","债","帐","斑","铃","旨","醇","董","饼","雏","姿","拌","傅","腹","妥","揉","贤","拆","歪","葡","胺","丢","浩","徽","昂","垫","挡","览","贪","慰","缴","汪","慌","冯","诺","姜","谊","凶","劣","诬","耀","昏","躺","盈","骑","乔","溪","丛","卢","抹","易","闷","咨","刮","驾","缆","悟","摘","铒","掷","颇","幻","柄","惠","惨","佳","仇","腊","窝","涤","剑","瞧","堡","泼","葱","罩","霍","捞","胎","苍","滨","俩","捅","湘","砍","霞","邵","萄","疯","淮","遂","熊","粪","烘","宿","档","戈","驳","嫂","裕","徙","箭","捐","肠","撑","晒","辨","殿","莲","摊","搅","酱","屏","疫","哀","蔡","堵","沫","皱","畅","叠","阁","莱","敲","辖","钩","痕","坝","巷","饿","祸","丘","玄","溜","曰","逻","彭","尝","卿","妨","艇","吞","韦","怨","矮","歇","郊","禄","捻","漠","粹","颠","宏","冤","肪","饥","呵","仙","押","挨","醛","娃","拾","没","佩","勿","吓","讹","侯","恋","夕","锌","篡","戚","淋","蓬","岂","釉","兆","泊","魂","拘","亡","杠","摧","氟","颂","浑","凌","铀","诱","犁","谴","颁","舶","扯","嘉","萌","犹","滋","焊","舌","匹","媳","肺","掠","酿","烹","疲","驰","鸦","窄","辱","狭","朴","遣","菲","奸","韧","辣","拳","秆","卧","醉","竭","茅","墓","矣","哎","艳","敦","舆","缔","雇","尿","葬","履","契","禽","渣","衬","躲","赔","咸","溉","贼","醋","堤","抖","妃","裤","廉","晴","挽","掀","茫","丑","亥","拦","悠","阐","慧","佐","奇","竖","孝","柜","麟","绣","遥","逝","愁","肖","昭","芬","逢","窑","捷","圜","盲","闸","宙","辐","披","账","狼","幽","绸","蜂","慎","餐","酬","誓","惟","叉","弥","址","帜","芝","砌","唉","仆","涛","臭","翠","盒","劫","慨","炳","阖","寂","椒","倘","拓","畏","喉","巾","颈","垦","拚","兽","蔽","芦","乾","爽","窃","谭","挣","崩","模","褐","传","翅","儒","伞","晃","谬","胚","剖","凑","眠","浊","霜","礁","蔑","抄","闯","洒","碑","蓉","耶","猜","蹲","壶","唤","澳","锯","郡","玲","绵","纽","梳","掏","吁","锤","鼠","穴","椅","殷","遮","吵","萍","厌","畜","俱","夸","吕","囊","捧","雌","闽","饶","瞬","郁","哨","凿","朝","俺","浒","茂","肝","勋","盯","籽","耻","菊","滥","稼","戒","奈","帅","鞭","蚕","镁","询","跌","烤","坛","宅","笛","鄂","蛮","颤","棍","睁","鼎","岌","降","侍","藩","嚷","匪","岳","糟","缠","迪","泄","卑","氛","堪","萝","盛","碘","缚","悦","澄","甫","攀","屠","溢","拱","晰","携","朽","吟","菱","谦","凹","俊","芒","盼","婶","艘","酰","趁","唇","挫","羞","浴","疼","萎","肴","愚","肿","刨","绞","枢","嫁","慕","舱","铲","苹","豫","谕","迭","潘","顷","翁","榜","匠","欠","茬","畴","胃","沾","踪","弊","哼","鹏","歧","桐","沃","悼","惑","溃","蔗","荐","潭","孢","露","诊","庸","聪","嫌","厨","庞","祁","钳","肆","梭","赠","崖","篮","颖","甸","藻","捣","且","撕","诏","贞","赐","慈","炕","胖","兹","差","琼","锈","汛","卓","棵","馈","挠","灶","婴","蒂","肤","衫","沥","仑","勉","沪","逸","蜜","浦","嗓","晕","膏","祭","赢","艾","扮","鹅","怜","蒲","兔","孕","呖","蘖","挪","淑","谣","惧","廊","缅","俘","骄","膀","陡","宰","诞","峻","恼","腺","猎","涡","夷","愉","魔","铵","葛","贾","似","荫","哟","脊","钞","苛","锰","椭","镶","杏","溴","倚","滞","会","氓","捏","斩","傲","匆","僵","卤","烫","衍","榨","拢","裸","屑","咽","坊","舅","渴","翔","邪","拄","窖","猫","砌","钦","媒","脾","勺","柏","栅","噪","昼","耿","扁","辰","秤","得","贩","糕","梁","昙","衷","宦","扔","哇","诈","嘱","藤","卜","冈","悔","廓","皂","拐","氰","杉","玛","矢","寓","瓣","罕","垮","笋","淘","衔","称","恭","喇","帕","桉","秉","帘","铭","蛇","摔","斋","叭","帆","裸","俭","瘤","篷","砸","肢","辟","脖","瞪","暑","卜","竿","歼","笙","酮","蕴","哗","瞎","喀","刃","楔","喘","枚","嵌","挝","厢","粤","甩","拴","膝","恳","腕","娓","熄","锚","忌","愧","哦","荆","圃","骚","丸","蒜","毯","弗","俯","鹿","梢","屯","衙","轿","贱","垒","谅","踢","哑","滔","渥","饷","泳","棕","熬","搁","腈","梨","吻","樱","奠","捆","姨","柏","聘","惕","郓","绑","冀","裹","酥","寡","彦","稠","啡","钝","汝","擅","汰","鳙","埔","敞","嘿","逊","栋","谨","咖","鲤","雀","佣","庵","葫","贿","鳞","拼","搏","谎","塌","忉","腻","戊","怖","坟","禾","刹","嘻","桔","坎","拇","煽","狮","痒","曾","梗","寇","鹰","烛","哄","莽","雯","胳","龟","亟","糠","泌","坪","傻","什","喻","渊","蚌","跪","巷","涅","钊","譬","蕊","膛","侮","奕","枕","辫","况","扼","郝","寥","凄","厦","腥","钧","耦","蹄","戥","屁","诵","匈","桩","钓","涵","倦","袍","抒","屿","蹈","忿","敷","虹","聊","嗣","尉","灿","糙","蹬","嗯","姬","狡","笨","辜","僧","茨","讽","翰","枉","岐","枣","崭","焚","咕","猴","揽","涝","耍","趟","汹","咋","傍","镀","给","爵","虏","劈","璋","踩","瞅","迄","昔","汞","呱","诡","魄","祺","嘲","惶","赃","癌","咐","歉","扳","鄙","庐","聂","便","芡","躯","贬","煌","拧","隋","襄","淤","宠","炊","滇","謇","懒","栓","佑","憾","骆","裙","猖","兜","孵","痼","盥","曝","泣","絮","韵","眷","旷","噢","参","栖","盏","鳌","溅","煎","校","榴","暮","琪","淆","陛","巢","哒","吼","槐","唧","其","沛","乞","蜀","蜇","赚","捍","铰","幂","尧","咒","耽","叮","褂","焕","煞","雹","搓","釜","铬","拣","募","淹","瑰","鲢","茄","灼","邹","躬","觉","娇","焉","彰","鹤","琳","沦","畔","惹","庶","毙","皖","邢","禹","渍","绷","窜","翘","淫","箪","陌","膊","鞑","咳","玫","巫","拂","蕉","澜","赎","绥","锄","囱","赌","颊","缕","寅","躁","稚","庚","苟","氦","魁","珊","蜕","蛭","酌","逗","闺","蔓","撇","豌","朕","缉","襟","镍","桅","荧","侄","卒","佃","瞿","娶","饪","耸","乍","靶","痴","靖","扛","筐","韶","嚣","崔","蓿","岔","氘","娥","剿","霖","喃","搪","雍","裳","撰","豹","骏","慷"] def static useDistKeys = [] static def totalCount = new AtomicLong() -//"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319", -// "安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","深圳":"sz|4","北京":"bj|1","上海":"sh|2","天津":"tj|18","重庆":"cq|37","广州":"gz|3","东莞":"dg|413","佛山":"fs|222", - def cityList = ["杭州":"hz|79", "成都":"cd|102","石家庄":"sjz|241","青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740", - "西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", - "淮北":"huaibei|9357","滁州":"chuzhou|10266", + + def cityList = ["福州":"fz|304","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","苏州":"su|5","无锡":"wx|93", "深圳":"sz|4","郑州":"zz|342", "长沙":"cs|414","东莞":"dg|413", + "保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", + "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", + "石家庄":"sjz|241","太原":"ty|740","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", + "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", + "淮北":"huaibei|9357","滁州":"chuzhou|10266", + "蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", + "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","潍坊":"wf|362", + "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", + "南通":"nt|394","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", + "呼和浩特":"hu|811", "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", - "厦门":"xm|606","泉州":"qz|291","莆田":"pt|2429","漳州":"zhangzhou|710", - "宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", + "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", - "漳浦":"zhangpu|717", "中山":"zs|771", - "珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791", - "肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303", + "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", + "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", - "南宁":"nn|845","柳州":"liuzhou|7133","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", + "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", - "贵阳":"gy|2015","遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", + "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", - "清镇":"qingzhen|12703", - "兰州":"lz|952","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", + "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", - "海口":"haikou|2053","三亚":"sanya|2422","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", + "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", - "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380", - "儋州":"danzhou|10394", - "郑州":"zz|342","洛阳":"luoyang|556","新乡":"xx|1016","南阳":"ny|592","许昌":"xc|977", - "平顶山":"pds|1005","安阳":"ay|1096","焦作":"jiaozuo|3266","商丘":"sq|1029","开封":"kaifeng|2342", - "濮阳":"puyang|2346","周口":"zk|933","信阳":"xy|8694","驻马店":"zmd|1067","漯河":"luohe|2347", + "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", + "儋州":"danzhou|10394", "新乡":"xx|1016","许昌":"xc|977", + "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", + "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", @@ -80,11 +85,10 @@ class Esf58Crawl extends BaseBatch{ "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", - "哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", + "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", - "肇州":"zhaozhou|382", - "武汉":"wh|158","宜昌":"yc|858","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", + "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", @@ -92,53 +96,48 @@ class Esf58Crawl extends BaseBatch{ "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", - "长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914", - "湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", + "益阳":"yiyang|10198","衡阳":"hy|914", + "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", - "保定":"bd|424","唐山":"ts|276","廊坊":"lf|772","邯郸":"hd|572", - "秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993","张家口":"zjk|3328", + "邢台":"xt|751","衡水":"hs|993", "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", "沙河":"shaheshi|755", - "徐州":"xz|471", - "南通":"nt|394","扬州":"yz|637","盐城":"yancheng|613","淮安":"ha|968","连云港":"lyg|2049", - "泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", + "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", - "金湖":"jinhu|975","江阴":"jiangyins|34984", - "南昌":"nc|669","赣州":"ganzhou|2363","九江":"jj|2247","宜春":"yichun|5709","吉安":"ja|2364", + "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", "浮梁":"fuliangxian|9071", - "长春":"cc|319","吉林":"jl|700","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", + "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", "梨树县":"lishu|10176", - "沈阳":"sy|188","鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722", + "鞍山":"as|523","抚顺":"fushun|5722", "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", - "呼和浩特":"hu|811","包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", + "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", - "西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", + "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", - "济南":"jn|265","烟台":"yt|228","临沂":"linyi|505", - "淄博":"zb|385","济宁":"jining|450","泰安":"ta|686","聊城":"lc|882","威海":"weihai|518", - "枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","东营":"dy|623","菏泽":"heze|5632", + "泰安":"ta|686","聊城":"lc|882", + "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", @@ -151,14 +150,14 @@ class Esf58Crawl extends BaseBatch{ "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", "临朐":"linqu|374","鄄城":"juancheng|5635", - "临汾":"linfen|5669","大同":"dt|6964","运城":"yuncheng|5653","晋中":"jz|8854", + "运城":"yuncheng|5653","晋中":"jz|8854", "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", - "咸阳":"xianyang|7453","宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", + "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", - "绵阳":"mianyang|1057","德阳":"deyang|2373","南充":"nanchong|2378","宜宾":"yb|2380", + "德阳":"deyang|2373","宜宾":"yb|2380", "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", @@ -170,18 +169,20 @@ class Esf58Crawl extends BaseBatch{ "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", - "昆明":"km|541","曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", + "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", - "宁波":"nb|135","温州":"wz|330","金华":"jh|531", - "台州":"tz|403","绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793", + "丽水":"lishui|7921","衢州":"quzhou|6793", "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", - "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" + "威海":"weihai|518","安阳":"ay|1096", + "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", + "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"aihkou|2053", + "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|27 6","商丘":"sq|1029","大同":"dt|6964", ] static main(args) { @@ -191,26 +192,38 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["郑州"] - cityList.each{city, str-> +// "宁波","佛山","南通","烟台","临沂","呼和浩特","海口","广州", + def citys = ["唐山","泰安","襄阳","台州","镇江","银川","江门","荆州","北京","衡阳","南京","滨州","张家口","商丘","赣州","湖州","安庆","聊城","枣庄","阜阳","西安","芜湖","吉林","遂宁","鞍山","东营","安阳","咸阳"] + for(def city : citys){ + def str = cityList.get(city) for(def i=0;i<1;i++){ -// def city = "福州" -// if(citys.contains(city)){ - def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); - if(districts){ - def arrs = str.split("\\|") - def url = "https://"+arrs[0]+".58.com/xiaoqu/" - distMap.put(city, districts) - - addCrawlKeys(city); - addDetailContent(city, url); - -// Thread.sleep(1000*60*10) - } -// } + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) + addCrawlKeys(city); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); + } } } +// cityList.each{city, str-> +// for(def i=0;i<1;i++){ +// if(citys.contains(city)){ +// def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); +// if(districts){ +// def arrs = str.split("\\|") +// def url = "https://"+arrs[0]+".58.com/xiaoqu/" +// distMap.put(city, districts) +// +// addCrawlKeys(city); +// addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); +// } +// } +// } +// } + } @@ -219,14 +232,18 @@ class Esf58Crawl extends BaseBatch{ for(def i=0;i5){ + districtName = districtName.substring(0,3) + } for(def j=1;j<=districtName.length()-1;j++) { - if(j<=2){ +// if(j<=3){ def keyword = districtName.toString().substring(0, j); - def keyStr = city+"##"+keyword; + def keyStr = keyword; if(!useDistKeys.contains(keyStr)){ useDistKeys << keyStr } - } +// } } } useDistKeys.sort {a,b-> @@ -239,25 +256,26 @@ class Esf58Crawl extends BaseBatch{ - def addDetailContent(def city, def url){ + def addDetailContent(def city, def url, def cityid){ final CountDownLatch latch = new CountDownLatch(useDistKeys.size()) for(def k=0;k0){ + def datas = jsonObj.data + for(def d : datas){ + def obj = findByParams(city, d.areaName, d.name); + def price = null + if(d.price){ + price = d.price.toInteger() + } + if(obj){ + DBObject udata2 = new BasicDBObject() + udata2.put("avg_price", price); + udata2.put("c_date", new Date()); + println "[update] "+city + " "+ d.name + thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2)); + }else{ + + println "[save] "+city + " "+ d.name + saveThirdParty(["url":d.listName,"city":city,"region":d.areaName,"block":null,"name":d.name,"address": d.address, "avg_price": price]); + } + } + } + }else{ + println "【"+keyword +"】 无结果" + } + + } + } + def addSearchContent(city, url, keyword){ def flag = true; @@ -284,9 +338,11 @@ class Esf58Crawl extends BaseBatch{ def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET); println "[${totalCount.get()}]关键字【${keyword}】 页:"+ newUrl if(document){ - Elements elements = JQueryUtils.select([".list-cell a"], document) - if(elements){ - saveDetailContent(city, document); + Elements infoElements = JQueryUtils.select([".list-cell .li-info"], document) + Elements priceElements = JQueryUtils.select([".list-cell .li-side"], document) + if(infoElements){ +// saveDetailContent(city, document); + saveLists(city, infoElements, priceElements) }else{ flag = false; } @@ -294,6 +350,46 @@ class Esf58Crawl extends BaseBatch{ flag = false; } page++; + if(page>50){ + flag = false; + } + } + } + + + def saveLists(city, infoElements, priceElements){ + for(def i=0;i") } - def avg_price = JQueryUtils.selectText(".price", detailDocument); + def avg_price = JQueryUtils.selectText(".average", detailDocument); def region; - def addr = JQueryUtils.selectText("td:contains(详细地址)+td", detailDocument); + def addr = JQueryUtils.selectText(".column-1:contains(小区地址) .value", detailDocument); if(info){ if(info.size()>=4){ - city = info[2].replace("小区","").trim() - region = info[3].replace("小区","").trim() + city = info[1].replace("小区","").trim() + region = info[2].replace("小区","").trim() } } if(!addr){ @@ -377,8 +473,8 @@ class Esf58Crawl extends BaseBatch{ obj.put("name", metadata.name); obj.put("s_date", SHORT_DATE); obj.put("source", SOURCE); - obj.put("addres", metadata.address); - obj.put("avg_price", metadata.avg_price.toInteger()); + obj.put("address", metadata.address); + obj.put("avg_price", metadata.avg_price); obj.put("c_date", new Date()); println obj.toString() thirdPartyColl.save(obj); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddMetadataBatchQfang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddMetadataBatchQfang.groovy index f6e5f8b79..9d1f5561f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddMetadataBatchQfang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddMetadataBatchQfang.groovy @@ -315,11 +315,11 @@ class AddMetadataBatchQfang{ private static void saveThirdPartyPrice(def saveDate){ def q = new BasicDBObject() - q.put("url", saveDate.get("xqUrl")) q.put("s_date", SHORT_DATE); q.put("city", saveDate.get("city")); q.put("region", saveDate.get("region")); q.put("name", saveDate.get("xqName")); + q.put("source", saveDate.get("source")); def udata = THIRD_PARTY_PRICE_COLL.findOne(q) if(!udata){ @@ -336,6 +336,14 @@ class AddMetadataBatchQfang{ obj.put("source", saveDate.get("source")); obj.put("c_date", new Date()); THIRD_PARTY_PRICE_COLL.save(obj); + println "[save] "+saveDate.toString() + }else{ + def updata = new BasicDBObject() + updata.put("avg_price", saveDate.get("price")) + updata.put("url", saveDate.get("xqUrl")); + updata.put("m_date", new Date()); + THIRD_PARTY_PRICE_COLL.update(q,new BasicDBObject('$set',updata)) + println "[update] "+saveDate.toString() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddSeedBatchQfang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddSeedBatchQfang.groovy index 3584a5201..b4911d45c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddSeedBatchQfang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddSeedBatchQfang.groovy @@ -87,7 +87,7 @@ class AddSeedBatchQfang extends BaseBatch{ href = "https:"+href; } if(href&&!CITYS.contains(city)){ - ret.put(city, href); + ret.put(city, href); } } }catch(Exception e){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index e870f26ab..b620f3098 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,9 +15,10 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); -// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", -// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] - def sourceMap = ["fang.com":"搜房"] +// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", +// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] + //"fang.com":"搜房", + def sourceMap = ["ke.com":"贝壳找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -25,7 +26,7 @@ class SynchDistrictPrice { def run(){ - def citys = CommonUtils.findAllCity() + def citys = ["重庆"]//CommonUtils.findAllCity() sourceMap.each { k, v -> for(def city : citys){ @@ -93,11 +94,16 @@ class SynchDistrictPrice { for(def r : rets){ def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) if(obj){ + updateIndex++ thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) - addIndex++ thirdPartyColl.save(r) }else{ - thirdPartyColl.save(r) + try{ + addIndex++ + thirdPartyColl.save(r) + }catch(Exception e){ + e.printStackTrace() + } } // if(!obj){ // addIndex++ @@ -111,7 +117,7 @@ class SynchDistrictPrice { // } // } } - println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex + println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 已存在:"+updateIndex } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index 80d802c97..4866f25b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -12,10 +12,10 @@ class TestCrawlTotalCount { task.run(); } def run(){ - def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"] + def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202109); - def count2 = findTotalCount(source, 202110); + def count1 = findTotalCount(source, 202203); + def count2 = findTotalCount(source, 202204); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index ddc282cb5..0b5a49b5d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -4,8 +4,10 @@ import com.fangjia.core.util.DataUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.OtherPriceUtils +import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -38,7 +40,6 @@ class ThirdDataToGatherBatch extends BaseBatch{ static main(args) { runBatch(new ThirdDataToGatherBatch(), args) } - @Override void process(Object args) { println SHORT_DATE diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy index a57c51cc7..fee1de15f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ZhugefangCrawl4.groovy @@ -278,7 +278,7 @@ class ZhugefangCrawl4 extends BaseBatch{ int goCount = 0; while(goCount < 10){ try{ - def rs = doPost("http://192.168.0.19:32222/",null, params, true, "utf-8") + def rs = doPost("http://192.168.0.216:32222/",null, params, true, "utf-8") if (rs) { JSONObject jsonObject = JSONObject.fromObject(rs); if(jsonObject.status_code==200){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy new file mode 100644 index 000000000..6540e9ca0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.thirdPriceCrawl.trade + +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.HttpClient +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.apache.http.Header +import org.apache.http.HttpResponse +import org.apache.http.client.methods.HttpGet +import org.apache.http.impl.client.HttpClients +import org.apache.http.client.HttpClient + +class LianJiaTradeCrawl { + + def static final LOGIN_URL = "https://clogin.lianjia.com/authentication/authenticate" + + public def CHARSET = "utf-8"; + public def ERROR_MSG = "访问过于频繁"; + + static main(args) { + def task = new LianJiaTradeCrawl() + task.run() + } + + def run(){ + def valueMap = getUidAndSid(); + def xqMap = goToXiaoqu(valueMap); + // goLogin() + } + + def getUidAndSid(){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("https://sh.lianjia.com/"); + get.setHeader("Accept-Encoding", "gzip, deflate, br") + get.setHeader("Accept-Language", "zh-CN,zh;q=0.9") + get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + HttpResponse response = client.execute(get); + def valueMap = getUidAndSid(response) + return valueMap; + + } + + def goToXiaoqu(valueMap){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("https://sh.lianjia.com/xiaoqu/"); + get.setHeader("Host","sh.lianjia.com") + get.setHeader("Connection","keep-alive") + get.setHeader("sec-ch-ua-mobile","?0") + get.setHeader("sec-ch-ua-platform","Windows") + get.setHeader("Upgrade-Insecure-Requests","1") + get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + get.setHeader("Sec-Fetch-Site","same-origin") + get.setHeader("Sec-Fetch-Mode","navigate") + get.setHeader("Sec-Fetch-User","?1") + get.setHeader("Sec-Fetch-Dest","document") + get.setHeader("Referer","https://sh.lianjia.com/") + get.setHeader("Accept-Encoding","gzip, deflate, br") + get.setHeader("Accept-Language","zh-CN,zh;q=0.9") + + def cookieStr = "" + valueMap.each {k,v-> + cookieStr = cookieStr +k +"="+v+";" + } + get.setHeader("Cookie", cookieStr) + + HttpResponse response = client.execute(get); + def newVvalueMap = getUidAndSid(response) + return valueMap; + } + + + def getUidAndSid(HttpResponse response) { + Header[] headers = response.getHeaders("Set-Cookie"); + def valueMap = [:] + for(def header : headers){ + String[] split = header.getValue().split(";"); + for (String s : split) { + def arrs = s.split("=") + valueMap.put(arrs[0], arrs[1]) + } + } + return valueMap; + } + + + + def goLogin() { + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","clogin.lianjia.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","1415") + headers.put("sec-ch-ua",'Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"') + headers.put("sec-ch-ua-mobile","?0") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + headers.put("sec-ch-ua-platform","Windows") + headers.put("Content-Type","application/json") + headers.put("Accept","*/*") + headers.put("Sec-Fetch-Site","same-origin") + headers.put("Sec-Fetch-Mode","cors") + headers.put("Sec-Fetch-Dest","empty") + headers.put("Cookie","Cookie: lianjia_uuid=45384ffe-4245-46fb-aa3f-aa6931a0a392; lianjia_ssid=75aa3a46-9d5f-42a9-b39b-4fc5e181e37c; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22%24device_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; select_city=350100; crosSdkDT2019DeviceId=-y4aedm--2s8wtc-85pqx15zlsvkewy-gkpcb9u2t") + + headers.put("Accept-Encoding", "gzip, deflate, br") + headers.put("Accept-Language", "zh-CN,zh;q=0.9") +// headers.put("User-Agent", UserAgents.get()) + headers.put("Referer", "https://clogin.lianjia.com/authentication/sdk/init?version=2.0") + headers.put("Origin", "https://clogin.lianjia.com") + //params信息 + Map params = new LinkedHashMap() + params.put("mainAuthMethodName", "username-password") + params.put("srcId", "eyJ0Ijoie1wiZGF0YVwiOlwiMWM1ZDg5ZTFjOTc0ZWViODliOTJkYTE1ZWEzOWM5ZDY1MWNlODJkMjkzYzQzYmY2NjRiMWEyYWI2NDRkYmViY2RiMDVkNmY3YTc3MmVkODBlZjNhMjBjYWM5ODVlNjY0ODc3YmVjMTBjYTRkM2RkNWFhOTY0Mjc0OTliNzVkZWU4ZDU1YzVkNzA1ZTRhMzcyY2U1MGZkMGVmZDE0OWY0NzYxMzk4MjRlMjU0Yzk0MzY0MDJjMjdkYjkxODdjNDliOWU4MWJlNWJjNGY2ZmEyNmY5Yjk2NzAxOTFkZDQwOWMxMzhjNTEwOTUwMmQ1OGE4NTE0Y2IzMzgxNzQ0ZTgwMzg2NmVjOGZhOWIyNjc1MzEyMTQ5YjE2NjU3ZjEyNDYyY2RiOTc4ZjM2ODVlNWVmYWViMGU1YjEwZmFiMWUzNDFhMzg4NmYzZmMwM2FhZWNjZDg1ZmIwNGYzZDQ2NmRjNlwiLFwia2V5X2lkXCI6XCIxXCIsXCJzaWduXCI6XCIzYjYwYTYwOFwifSIsInIiOiJodHRwczovL2Nsb2dpbi5rZS5jb20vbG9naW4/c2VydmljZT1odHRwcyUzQSUyRiUyRnd3dy5rZS5jb20lMkZ1c2VyJTJGY2hlY2tsb2dpbiUzRnJlZGlyZWN0JTNEaHR0cHMlMjUzQSUyNTJGJTI1MkZmei5rZS5jb20lMjUyRmNoZW5namlhbyUyNTJGYzM0MjAwMzI1MDU1MzAxMDElMjUyRiIsIm9zIjoid2ViIiwidiI6IjAuMSJ9") + params.put("version", "2.0") + params.put("accountSystem", "customer") + params.put("service", "https://www.lianjia.com/user/checklogin?redirect=https%3A%2F%2Ffz.lianjia.com%2Fchengjiao%2F") + params.put("loginTicketId", "nM7eXE8ZWBP67aiapHqZRpn0QHTfCX4q") + + + Map credentialparams = new LinkedHashMap() + credentialparams.put("username", "15021384995") + credentialparams.put("password", "dx/vjEHyEc8ETOqKDn5ZhNQZuO5JjoCu1D0UXVTQ2UrOE/liQYeNcj9H9bPB5c43UDRyKZSvA9UzBc4ycTOkDkq7pgITkJrzP0Iagkj19R0TT/yX/9NHEwm/GaqBfVvfJXfmw6ZX9vCYLaOaiOJ31PTpI5hDUS/PxxHBkmqC7Bg=") + credentialparams.put("ioaRunState", false) + credentialparams.put("ioaMid", "{}") + credentialparams.put("alertSt", "") + credentialparams.put("ioaStateVersion", 2) + credentialparams.put("encodeVersion", "1") + params.put("credential", credentialparams) + + Map contextparams = new LinkedHashMap() + contextparams.put("deviceId", "default") + contextparams.put("sign", "default") + params.put("context", contextparams) + + //post请求 + responseMap = isProxyAndTryTimesToRequest(LOGIN_URL, HttpClient.CHARSET, 100, "POST", params, headers, true) + } catch (Exception e) { +// println "goLogin(${username}, ${password}, ${rememberme})发生异常, 异常信息为:" + e.getMessage() + } + return responseMap + } + + + def static isProxyAndTryTimesToRequest(url, charset, tryTimes, requestType, params, headers, isJson) { + def responseMap = [:] + def _tryTimes_ = 0 + //请求响应为空(切换代理重试) + while (_tryTimes_ < tryTimes) { + try { + //第一次请求不提示重试 +// println("请求地址URL为:" + url + "代理尝试次数tryTimes为" + (_tryTimes_ + 1)) + if (StringUtils.isNotEmpty(requestType) && requestType.equals("POST")) { + responseMap = ABuYunUtil.doPostResponse(url, params, headers, charset, isJson) + } else { + responseMap = ABuYunUtil.doGetResponse(url, params, headers, charset) + } + def response = responseMap.get("response") + if (StringUtils.isEmpty(response)) { +// println('isProxyAndTryTimesToRequest 请求response为空') + //控制sleep时间 + //Thread.sleep(sleepTimes); + } else { + //控制sleep时间 + //Thread.sleep(sleepTimes); +// println('isProxyAndTryTimesToRequest 请求response响应成功, 响应内容为:' + response) + break + } + } catch (Exception e) { +// println("isProxyAndTryTimesToRequest请求发生异常, 请求url为: " + url + "请求异常信息为:" + e.getMessage()) + } finally { + //统计所有请求总次数(后期统计用) +// println("目前整个" + GLOBAL_WEBSITE_SOURCE + "抓取请求总次数count为: " + count.incrementAndGet()) + _tryTimes_++ + } + } + responseMap.put("tryTimes", _tryTimes_) + responseMap.put("url", url) + return responseMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index 3d1d74c0c..21887e3db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -167,7 +167,16 @@ public class CommonUtils { } - + static void sendMessage(String queueName, byte[] bytes){ + ConnectionFactory factory = new ConnectionFactory(); + factory.setUri("amqp://guest:guest@192.168.0.100:35677/%2f"); + Connection conn = factory.newConnection(); + Channel channel = conn.createChannel(); + channel.queueDeclare(queueName, true, false, false, null); + channel.basicPublish("", queueName, null, bytes); + channel.close() + conn.close() + } /** @@ -236,6 +245,48 @@ public class CommonUtils { } + + static void recalculationSqRent(String city, String region, String name, String estateType, Date calcDate = new Date(), boolean isTest = false, int calcType = 0, needRedrawTrend = true, def sync = false) { + + + calcDate = com.fangjia.core.util.DateUtil.truncateDay(calcDate) + if (estateType && estateType.contains("别墅")) { + estateType = "别墅" + } else if (estateType) { + estateType = "小区" + } + + DBObject cmd = new BasicDBObject() + cmd.put("city", city) + cmd.put("region", region) + cmd.put("name", name) + cmd.put('estate_type', estateType) + cmd.put("calc_date", calcDate) + cmd.put("calc_type", 0) + println(cmd) + DBObject data = new BasicDBObject() + data.put("category", "square_rent") + data.put("priority", 3) + data.put("status", 0) + data.put("remark", 'week') + data.put("c_date", new Date()) + + + if(isTest){ + data.put("ip", "192.168.0.231") + COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false) + }else{ + data.put("ip", ["fangjia-priceanalysis-0", "fangjia-priceanalysis-1"][new Random().nextInt(2)]) + COMMAND_COLL.update(cmd, new BasicDBObject('$set', data), true, false) + } + if(sync){ + while (COMMAND_COLL.count(cmd) != 0){ + Thread.sleep(1000) + } + } + + } + /** * 重算小区均价 * @param city diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy index bfa7f8ec7..f6399a2dd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DateUtils.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.util +import org.apache.commons.lang.StringUtils + import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Calendar; @@ -995,4 +997,42 @@ class DateUtils { return rtn; } + public static Date parseString2Date(String str){ + if(StringUtils.isBlank(str)){ + return null + } + str = str.replaceAll("^\\s+", "").replaceAll("\\s+\$", "") + + Date now = new Date() + if(str.contains("小时")||str.contains("分钟") || str.contains("秒") || str.contains("当前")|| str.contains("今天")){ + return org.apache.commons.lang3.time.DateUtils.truncate(new Date(), Calendar.DAY_OF_MONTH) + }else if(str.contains("昨天")) { + return add(now, -1) + }else if(str.contains("天")){ + int days = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*days) + }else if(str.contains("月前")){ + int month = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*month*30) + }else if(str.contains("年前")){ + int years = Integer.parseInt(str.replaceAll("[^\\.0123456789]", "")) + return add(now, (-1)*years*365) + }else{ + if(str.size()<10){ + if(str.contains("-")){ + str = (now.getYear()+1900)+"-"+str + } + }else{ + if(str.contains("月") && str.contains("日") && !str.contains("年")){ + str = (now.getYear()+1900)+"年"+str.substring(0, str.indexOf("日")+1) + } + } + if(str.contains("年")){ + return DateUtils.parseFromTightByPattern(str, "yyyy年MM月dd日") + }else{ + return DateUtils.parseFromTightByPattern(str, "yyyy-MM-dd") + } + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy index 3b83880cd..7e6e73263 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/DbUtils.groovy @@ -92,11 +92,6 @@ public class DbUtils{ } } - - public static void executeQueryOnSeawater(BasicDBObject query,Closure c, BasicDBObject fields = null){ - seawater.find(query,fields).batchSize(10000).each(c) - } - public static void executeQueryOnSeaweed(BasicDBObject query,Closure c, BasicDBObject fields = null){ SEAWEED_SLAVE.find(query,fields).batchSize(1000).each(c) } @@ -171,10 +166,6 @@ public class DbUtils{ return SEAWEED_SLAVE.findOne(query,fields); } - public static int executeCountOnSeawater(BasicDBObject query){ - return seawater.count(query); - } - public static BasicDBObject buildSeaweedQuery(def city = null, def region = null, def name = null, def cat = "district", def visible = 0){ BasicDBObject query = new BasicDBObject() if(city){ @@ -314,6 +305,43 @@ public class DbUtils{ } return districts + } + + /** + * 查询附近一共里内小区的数量 + */ + def static findAroundDistrictByRadius(BasicDBObject district, def radius){ + def districts = [] + if(hasGeo(district)){ + def query = new BasicDBObject() + query.append("cat", "district") + query.append("city", district.city) + query.append("visible", 0) + DBObject sdq = new BasicDBObject(); + def lat = district.lat / 10000000000.0 + def lng = district.lng / 10000000000.0 + sdq.put("sd.point", lat + "," + lng); + sdq.put("sd.sfield", "lat_lng"); + sdq.put("sd.distance", String.valueOf((double) radius / 1000.0)); + query.append('$sd', sdq) + + def keys = new BasicDBObject() + keys.append("city", 1) + keys.append("region", 1) + keys.append("name", 1) + keys.append("lat", 1) + keys.append("lng", 1) + keys.append("lat2", 1) + keys.append("lng2", 1) + keys.append("estate_type2", 1) + DbUtils.SEAWEED_SLAVE.find(query, keys).each {BasicDBObject dbo-> + if(!district._id || dbo._id != district._id){ + districts << dbo + } + } + } + return districts + } /** * 查询小区均价 @@ -350,31 +378,6 @@ public class DbUtils{ return district.lat && district.lng && district.lng > 0 && district.lat > 0 } -/** - * 记录修改小区库的操作日志 - * @param city - * @param region - * @param name - * @return - */ - def static saveDeleteLog(String city, String region,String name,String category,int status){ - if(status!=1){ - return - } - DBObject obj = new BasicDBObject() - obj.put("city", city); - obj.put("region", region); - obj.put("district_name", name); - obj.put("category" , category) - obj.put("origin" , "脚本") - if(status==1){ - obj.put("type" , "remove") - } - obj.put("c_date" , new Date()) - obj.put("m_date" , new Date()) - OPERATION_LOG.save(obj) - } - public static DBCollection getCollection(String host, int port, String dbName ,String collName){ def collection = COLL_CACHE["${host}#${port}#${dbName}#${collName}"] if(!collection){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index e6f062685..1e51359de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -1,12 +1,15 @@ package com.fangjia.script.util import com.fangjia.script.lixue.util.DateFormat +import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang.StringUtils import org.apache.poi.hssf.record.BoolErrRecord - +import org.apache.poi.hssf.usermodel.HSSFBorderFormatting import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.apache.poi.ss.formula.eval.ErrorEval import org.apache.poi.ss.usermodel.* +import org.apache.poi.ss.util.CellRangeAddress +import org.apache.poi.ss.util.RegionUtil import org.apache.poi.xssf.streaming.SXSSFWorkbook import java.util.concurrent.ExecutorService @@ -83,9 +86,6 @@ public class ExcelUtils{ } } ret.add(rowData); - if (ret.size() % 10000 == 0) { - println(ret.size()) - } } } catch (IOException e) { e.printStackTrace(); @@ -240,7 +240,7 @@ public class ExcelUtils{ return super.getProperty(propertyName) } - public static void write(String fileName, List> data, String sheetName){ + public static void write(String fileName, List> data, String sheetName, List> mergeData){ File file = new File(fileName) File dir = file.getParentFile(); @@ -309,6 +309,8 @@ public class ExcelUtils{ throw e } } + mergeCell(workBook, mainSheet, mergeData) + for(int i = 0; i < maxColumnNum; i++ ){ mainSheet.autoSizeColumn(i) } @@ -326,6 +328,36 @@ public class ExcelUtils{ fos.close(); } + static void write(String fileName, List> data, String sheetName){ + write(fileName, data, sheetName, null) + } + + /** + * 合并单元格 + * @param workBook + * @param sheet + * @param data int firstRow, int lastRow, int firstCol, int lastCol + */ + static mergeCell(Workbook workBook, Sheet sheet, List> data) { + if (CollectionUtils.isEmpty(data)) { + return + } + //设置style + CellStyle mergedCellStyle = workBook.createCellStyle() + mergedCellStyle.verticalAlignment = VerticalAlignment.CENTER; + mergedCellStyle.alignment = HorizontalAlignment.CENTER; + + for (List rowCell : data) { + if (rowCell.get(0) != rowCell.get(1)) { + CellRangeAddress cra = new CellRangeAddress(rowCell.get(0), rowCell.get(1), rowCell.get(2), rowCell.get(3)) + sheet.addMergedRegion(cra); + //设置合并后style + def cell = sheet.getRow(rowCell.get(0)).getCell(rowCell.get(2)); + cell.setCellStyle(mergedCellStyle) + } + } + } + public static newWorkBook(String fileName){ Workbook wb; if(fileName.endsWith(".xlsx")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 5ef52da25..3d4e855f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -229,6 +229,153 @@ public class OpenApiClient { return null } + public static JSONObject searchLoudong(String id, String houseNum, String unitNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/property/searchLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject searchHouseLoudong(String id, String houseNum, String unitNum, String roomNum){ + try{ + Map params = new HashMap(); + params.put("id", id); + params.put("houseNumber", houseNum); + params.put("unitNumber", unitNum); + params.put("roomNumber", roomNum); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://localhost:8080/fangjia-open/property/searchHouseLoudong", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/searchHouseLoudong", params, "utf-8") + println("${id} ${houseNum} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${id} ${houseNum}") + } + } + + } catch (Exception e) { + println("${id} ${houseNum} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + + public static JSONObject webankEvaluate(String city, String region, String address, String area){ + try{ + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("address", address); + params.put("area", area.toString()); + params.put("token", TOKEN); + String content = new HttpRequest().get("http://open.fangjia.com/webank/evaluate", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/webank/evaluate", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if(result){ + if (result.getInt("code") == 200) { + return result; + } else if (result.getInt("code") !=0) { + println("${city} ${address} 程序调用异常") + } + } + + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + + public static JSONObject standard2Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/v2/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/v2/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 0) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + + public static JSONObject standard1Action(String city, String address) { + try { + Map params = new HashMap(); + params.put("city", city); + params.put("address", address); + params.put("token", TOKEN); + params.put("__refresh_cache","1") + + String content = new HttpRequest().get("https://open.fangjia.com/address/standard", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + while (result == null) { + result = HttpRequestUtil.getJsonContent("https://open.fangjia.com/address/standard", params, "utf-8") + println("${city} ${address} 程序调用异常") + Thread.sleep(6000) + } + if (result.getInt("code") == 200) { + return result.getJSONObject("result"); + } else if (result.getInt("code") !=0) { + println("${city} ${address}") + } + } catch (Exception e) { + println("${city} ${address} ${e.getMessage()}") + e.printStackTrace(); + } + return null + } + + public static JSONObject standardize(String city, String address) { try { Map params = new HashMap(); @@ -593,6 +740,91 @@ public class OpenApiClient { } + static JSONObject facilitys(String city, String region, String name) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("region", region) + params.put("name", name) + try { + String content = new HttpRequest().get("http://open.fangjia.com/district/facilitys", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject searchPaimais(String city, String region, String name, String startDate,limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city) + params.put("district", region) + params.put("name", name) + params.put("startDate", startDate) + params.put("limit", "100") + try { + String content = new HttpRequest().get("http://open.fangjia.com/auction/searchPaimais", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject searchDealPrice(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://open.fangjia.com/dealPrice/searchList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + static JSONObject houseList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startTime", startDate) + params.put("pageSize", limit.toString()) + try { + String content = new HttpRequest().get("http://open.fangjia.com/house/list", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + + + static JSONObject evaluaionList(String id, String startDate, limit) { + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("id", id) + params.put("startDate", startDate) + params.put("limit", limit.toString()) + try { + String content = new HttpRequest().get("http://open.fangjia.com/evaluation/bsList", params, "utf-8") + JSONObject result = JSONObject.fromObject(content); + return result + } catch (Exception e) { + + } + return null + } + static JSONObject districtDetail(String city, String region, String name) { Map params = new HashMap(); params.put("token", TOKEN); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy index a2026322b..278aced74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy @@ -179,31 +179,33 @@ public class KeywordSearchUtils{ public static final Map KEYWORDS_MAP = new HashMap(){{ put("地铁", "150500"); put("公交", "1507"); - put("码头", "1503"); - put("机场", "150104|150106"); - put("火车站", "150200"); put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); put("写字楼", "120201"); - put("停车场", "150903|150904|150905|150906"); - put("网吧", "080308"); put("学校","141200|141201|141202|141203|141204"); put("银行","1601|1603"); - put("公园/绿地","110100|110101"); - put("KTV","080302"); - put("健身中心","080111"); put("大型超市","0604"); - put("咖啡厅","0505"); - put("剧场/影院","080601|080603"); put("商场","0601"); - put("酒店","10"); - put("整容整形","090201"); - put("洗浴中兴","0714"); - put("便利店","0602"); put("餐饮","0501|0502|0503|0504"); - put("游乐场","080501"); put("旅游景点","110000"); - put("体育场馆","0800|0801"); put("图书馆","140500"); + +// put("码头", "1503");1 +// put("机场", "150104|150106");1 +// put("火车站", "150200");1 +// put("停车场", "150903|150904|150905|150906");1 +// put("网吧", "080308");1 +// put("公园/绿地","110100|110101"); +// put("KTV","080302");1 +// put("健身中心","080111");1 +// put("咖啡厅","0505");1 +// put("剧场/影院","080601|080603");1 +// put("酒店","10");1 +// put("整容整形","090201");1 +// put("洗浴中兴","0714");1 +// put("便利店","0602");1 +// put("游乐场","080501");1 +// put("体育场馆","0800|0801");1 +// put("住宅区","120300|120301|120302|120303|120304"); }}; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 38f6a9ad7..b7fe9aa99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20211001", "yyyyMMdd") -def endDate = DateUtil.parseDate("20211101", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220401", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220501", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy index fe4bad148..759375bf1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20211001", "yyyyMMdd") -def endDate = DateUtil.parseDate("20211101", "yyyyMMdd") +def startDate = DateUtil.parseDate("20211101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20211201", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java new file mode 100644 index 000000000..129465804 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -0,0 +1,379 @@ +package com.fangjia.script.statistics; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.DefaultHttpClient; + +import com.alibaba.fastjson.JSONObject; + +/** + * 蚂蚁分工开放平台工具类 + * 由于groovy调用会误报签名错误问题,故用 java 方式 + */ +public class ApiUtil { + + private static final String secretKey = "HtRc7R5jI0plOOAukES4pM6yxWJbU9GyyM1xi4kH"; + private static final String corpId = "ding0243a5c51f1f4aa8"; + + public static String getApiSingnature(String body, Long timeStamp, String secretKey) throws Exception { + JSONObject bodyObj = JSONObject.parseObject(body); + return getApiSingnature(bodyObj, timeStamp, secretKey); + } + + /** + * 获取签名 + */ + public static String getApiSingnature(JSONObject body, Long timeStamp, String secretKey) throws Exception { + String signature = ""; + try { + //参数排序,保证签名一致 + List fileds = new ArrayList<>(); + Set> set = body.entrySet(); + for (Entry entry : set) { + fileds.add(entry.getKey()); + } + //参数执行排序 + Collections.sort(fileds); + Map params = new LinkedHashMap(); + for (String key : fileds) { + Object obj = body.get(key); + if(obj != null){ + params.put(key, obj); + } + } + String stringToSign = "timestamp="+timeStamp+"&body="+ params.toString(); + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secretKey.getBytes("UTF-8"), "HmacSHA256")); + byte[] signatureBytes = mac.doFinal(stringToSign.getBytes("UTF-8")); + signature = new String(Base64.encodeBase64(signatureBytes)); + return signature; + } catch (Exception e) { + e.printStackTrace(); + throw new Exception("计算签名错误"); + } + } + + public static JSONObject post(String url, JSONObject json) throws Exception { + return (JSONObject) post(url, json, new JSONObject()); + } + + + private static Object post(String url, JSONObject json, Object t) throws Exception { + HttpClient client = new DefaultHttpClient(); + HttpPost post = new HttpPost(url); + + post.setHeader("Content-Type", "application/json"); + String result = ""; + try { + StringEntity s = new StringEntity(json.toJSONString(), "utf-8"); + post.setEntity(s); + + // 发送请求 + HttpResponse httpResponse = client.execute(post); + + // 获取响应输入流 + InputStream inStream = httpResponse.getEntity().getContent(); + BufferedReader reader = new BufferedReader(new InputStreamReader(inStream, "utf-8")); + StringBuilder strber = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null){ + strber.append(line + "\n"); + } + inStream.close(); + result = strber.toString(); + if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + return JSONObject.parseObject(result, t.getClass()); + } else { + return null; + } + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("请求异常"); + } + } + + /** + * 获取项目标签 + * + * @return + */ + public static List listProjectLabel(String projectId) { + List labels = null; + try { + JSONObject body = new JSONObject(); + body.put("id", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/project/getProjectLabel", param); + if ((Integer)result.get("code") == 0) { + JSONArray jsonArray = result.getJSONArray("data"); + labels = jsonArray.toJavaList(ProjectLabel.class); + } + } catch (Exception e) { + e.printStackTrace(); + } + return labels; + } + + /** + * 获取项目列表 + * @return + */ + public static List listProject() { + List project = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("http://api.antbim.net/ant_api/project/getProjectList", param); + if ((Integer)result.get("code") == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + project = jsonArray.toJavaList(Project.class); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return project; + } + + /** + * 获取项目下任务清单列表 + * + * @return + */ + public static JSONObject listInventory(String projectId) { + JSONObject result = new JSONObject(); + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + body.put("projectId", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + result = ApiUtil.post("https://api.antbim.net/ant_api/project/inventory/getInventoryList", param); + } catch (Exception e) { + e.printStackTrace(); + } + return result; + } + + + /** + * 获取项目下任务类型列表 + * @return + */ + public static JSONObject listTaskType(String projectId) { + JSONObject result = new JSONObject(); + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + body.put("projectId", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + result = ApiUtil.post("https://api.antbim.net/ant_api/project/tasktype/getTasktypeList", param); + } catch (Exception e) { + e.printStackTrace(); + } + return result; + } + + /** + * 获取项目内任务列表 + * + * @return + */ + public static List listTaskByProject(String projectId, String startTime, String endTime, int limitStart, int limitNum) { + List tasks = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", limitStart); + body.put("limitNum", limitNum); + body.put("projectId", projectId); + body.put("createtimeStart", startTime); + body.put("createtimeEnd", endTime); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByProject", param); + if ((Integer)result.get("code") == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + tasks = jsonArray.toJavaList(Task.class); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return tasks; + } + + /** + * 获取人员任务列表 + */ + public static List listTaskByPerson(String userid, String startTime, String endTime, int limitStart, int limitNum) { + List tasks = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", limitStart); + body.put("limitNum", limitNum); + body.put("userid", userid); + body.put("type", ""); + body.put("timezone", ""); + body.put("createtimeStart", startTime); + body.put("createtimeEnd", endTime); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByPerson", param); + int code = (Integer)result.get("code"); + if (code == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + tasks = jsonArray.toJavaList(Task.class); + } + } else if (code == 90001){ + throw new RuntimeException("每个企业调用单个接口的频率不可超过60次/分钟,会返回错误码90001:listTaskByPerson"); + } else if (code == 90009) { + throw new RuntimeException("每个企业单个接口每天的最大上限为1000次,若超出会返回错误码90009:listTaskByPerson"); + } + } catch (Exception ex) { + ex.printStackTrace(); + } + return tasks; + } + + /** + * 获取人员任务列表 + */ + public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { + return listTaskByPerson(null, startTime, endTime, limitStart, limitNum); + } + /** + * 获取任务详情 + * + * @return + */ + public static TaskDetail getTaskDetail(String taskId) { + TaskDetail detail = null; + try { + JSONObject body = new JSONObject(); + body.put("id", taskId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskDetail", param); + int code = (Integer)result.get("code"); + if (code == 0) { + JSONObject obj = JSONObject.parseObject(String.valueOf(result.get("data"))); + detail = obj.toJavaObject(TaskDetail.class); + } else if (code == 90001){ + Thread.sleep(1000 * 60); + return getTaskDetail(taskId); + //throw new RuntimeException("每个企业调用单个接口的频率不可超过60次/分钟,会返回错误码90001:getTaskDetail"); + } else if (code == 90009) { + throw new RuntimeException("每个企业单个接口每天的最大上限为1000次,若超出会返回错误码90009:getTaskDetail"); + } + } catch (Exception e) { + //e.printStackTrace(); + } + return detail; + } + + public static void main(String[] args) { + //List projects = ApiUtil.listProject(); + //System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); + //ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); + System.out.println(ApiUtil.getTaskDetail("be85282dc57d46e3a36fc0801107e605")); + //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); + //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); +// int limitStart = 0; +// List tasks = ApiUtil.listTaskByPerson("055551305431136413","2021-03-01 00:00:00", "2021-04-30 23:59:59", limitStart, 50); +// List tempTasks = tasks; +// +// while (tasks != null && tempTasks != null && tempTasks.size() == 50) { +// limitStart += 50; +// tempTasks = ApiUtil.listTaskByPerson("055551305431136413","2021-03-01 00:00:00", "2021-04-30 23:59:59", limitStart, 50); +// if (CollectionUtils.isNotEmpty(tempTasks)) { +// tasks.addAll(tempTasks); +// } +// } +// for (Task task : tasks) { +// if ("穆良书".equals(task.getResponse().getName())) { +// System.out.println(task); +// } +// } + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java new file mode 100644 index 000000000..e5ea5b6a9 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java @@ -0,0 +1,92 @@ +package com.fangjia.script.statistics; + +/** + * @author shenjian + * @since 2021/5/8 + */ +public class Project { + private Integer taskCount; + private Integer taskendCount; + private String name; + private CreateUser createUser; + private String id; + private String state; + private String createDate; + + public Integer getTaskCount() { + return taskCount; + } + + public void setTaskCount(Integer taskCount) { + this.taskCount = taskCount; + } + + public Integer getTaskendCount() { + return taskendCount; + } + + public void setTaskendCount(Integer taskendCount) { + this.taskendCount = taskendCount; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public CreateUser getCreateUser() { + return createUser; + } + + public void setCreateUser(CreateUser createUser) { + this.createUser = createUser; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getCreateDate() { + return createDate; + } + + public void setCreateDate(String createDate) { + this.createDate = createDate; + } +} + +class CreateUser { + private String name; + private String userid; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUserid() { + return userid; + } + + public void setUserid(String userid) { + this.userid = userid; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java new file mode 100644 index 000000000..bb5864998 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java @@ -0,0 +1,57 @@ +package com.fangjia.script.statistics; + +/** + * @author shenjian + * @since 2021/5/8 + */ +public class ProjectLabel { + + private String color; + private String name; + private Groups groups; + private String id; + + public String getColor() { + return color; + } + + public void setColor(String color) { + this.color = color; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Groups getGroups() { + return groups; + } + + public void setGroups(Groups groups) { + this.groups = groups; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} + +class Groups { + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java new file mode 100644 index 000000000..78fd1a825 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java @@ -0,0 +1,75 @@ +package com.fangjia.script.statistics; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/8 + */ +@NoArgsConstructor +@Data +public class Task { + + @JSONField(name = "code") + private String code; + @JSONField(name = "response") + private ResponseDTO response; + @JSONField(name = "planBeginTime") + private String planBeginTime; + @JSONField(name = "name") + private String name; + @JSONField(name = "planEndTime") + private String planEndTime; + @JSONField(name = "id") + private String id; + @JSONField(name = "state") + private String state; + @JSONField(name = "inventory") + private InventoryDTO inventory; + @JSONField(name = "flow") + private FlowDTO flow; + @JSONField(name = "createDate") + private String createDate; + @JSONField(name = "project") + private ProjectDTO project; + + @NoArgsConstructor + @Data + public static class ResponseDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } + + @NoArgsConstructor + @Data + public static class InventoryDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "id") + private String id; + @JSONField(name = "abbrev") + private String abbrev; + } + + @NoArgsConstructor + @Data + public static class FlowDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } + + @NoArgsConstructor + @Data + public static class ProjectDTO { + @JSONField(name = "id") + private String id; + @JSONField(name = "proname") + private String name; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java new file mode 100644 index 000000000..3d0d4acfe --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java @@ -0,0 +1,67 @@ +package com.fangjia.script.statistics; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/8 + */ +@NoArgsConstructor +@Data +public class TaskDetail { + + @JSONField(name = "code") + private String code; + @JSONField(name = "realityBeginDate") + private String realityBeginDate; + @JSONField(name = "assignDate") + private String assignDate; + @JSONField(name = "flowConten") + private String flowConten; + @JSONField(name = "description") + private String description; + @JSONField(name = "realityEndDate") + private String realityEndDate; + @JSONField(name = "workTime") + private String workTime; + @JSONField(name = "confirmDate") + private String confirmDate; + @JSONField(name = "response") + private ResponseDTO response; + @JSONField(name = "name") + private String name; + @JSONField(name = "inventoryId") + private String inventoryId; + @JSONField(name = "state") + private String state; + @JSONField(name = "id") + private String id; + @JSONField(name = "planEndDate") + private String planEndDate; + @JSONField(name = "planBeginDate") + private String planBeginDate; + @JSONField(name = "flow") + private FlowDTO flow; + @JSONField(name = "createDate") + private String createDate; + + @NoArgsConstructor + @Data + public static class ResponseDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } + + @NoArgsConstructor + @Data + public static class FlowDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java new file mode 100644 index 000000000..d22c4a364 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java @@ -0,0 +1,31 @@ +package com.fangjia.script.statistics; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/10 + */ +@NoArgsConstructor +@Data +public class TaskPersonDTO implements Comparable { + + private String name; + private String project; + private String taskName; + private Double score; + private Double workTime; + + @Override + public int compareTo(TaskPersonDTO o2) { + if (this.getName().equals(o2.getName())) { + if (this.getProject() != null) { + return this.getProject().compareTo(o2.getProject()); + } + } else { + return this.getName().compareTo(o2.getName()); + } + return 0; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json new file mode 100644 index 000000000..64af63aad --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json @@ -0,0 +1,42 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "id": 59151, + "name": "凌高小区", + "imgsUrl": "[\"https://image1.ljcdn.com/hdic-resblock/5f42d5a7-acd9-4a6f-a9f9-92dd651adf6c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ff4c17ac-759a-4e52-bf24-561fa02e851a.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e1ace69b-ce4b-43f9-9e5d-b14244d510cc.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/359a3b16-9485-49ef-b1eb-2cae27751622.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0a17322f-4cdd-43b0-aec0-2d6114a5d636.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/174aac59-4da4-4e88-ad6c-d76a17a59e17.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a6c09e7a-8e96-48c3-b291-20603d61355f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ac1cb228-197a-4e60-b0b0-1bc1c451d00e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e4e9ec1b-5430-4269-b5f7-eb934e189e85.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d9e874a1-df15-42de-b5e7-fb369db97723.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e620cc27-d24b-4cf1-ab4e-d6b3467c180d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b3c00e2e-bacc-4683-abab-8da0a6395972.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b77c7777-41cd-44f0-969f-7d7083d402fe.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/712a3c9a-0d43-410b-becb-ebe47e62fc6d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c44cee38-afbe-42b3-bdf7-f3875a5d0837.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/288933d6-5f8c-4510-8396-15acd8f4580c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c1611809-a633-470c-871c-3a2862e50c79.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/07096876-dd38-415a-ac95-27b79452f664.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae7ffa23-426e-42b7-a509-1a6752c95b72.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0fa04835-9077-4ab6-9e24-72955db66906.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/eb7e289a-8e03-4067-81fa-c633d1a429a5.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/deeade88-3b91-4702-9fdf-68b556ec9921.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/75270a55-bef0-4274-8275-825abae9dce3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0dec445b-f47f-4f2f-996f-c77915d52673.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f05af36d-af7e-4d3f-a8ea-8f23536d2d23.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/40fa734e-682d-4ef6-8094-37e5d281a75f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a3e582a6-19d0-4eeb-9735-2a5a7916187e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f56baa4b-d531-4782-a619-9b22a44b8663.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cc14001f-08ce-4769-9c14-ef4ce1e1f406.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/1153dabe-4ecc-4246-aad2-23ac4575f67f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0712ae4-63dc-477f-9335-1e7147c201d1.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4051310b-ee71-489c-af36-dbb2dc8dd4e0.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/272b80dd-5356-4e7a-beaa-c9d2908d6214.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c238f4da-acc3-4abd-bd62-4c43d59aa46b.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae234e8e-d1d7-4a96-ba27-a333364ae907.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7dec0fa6-508b-4220-b20b-77560fe72c03.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/70cbd711-3bd8-4e6b-9b5e-a13ca6b41cfb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6a258444-96ee-4200-820a-449bb611c801.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0983610c-8ead-472c-a70b-9c50ede9cfd8.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/461dbab6-9cc2-4436-a466-3a06fb1bfe1e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7df6667a-948c-435b-81cb-c1c798614b2e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/97feda14-4672-481f-b127-c28b185cd0ca.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a131d9ca-efd3-4609-a43c-d32978d99cf3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/efd8958c-2108-4660-b8c1-611dad53abf2.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/93c57a6a-0832-4ea0-8512-fcf0d9f5d751.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0dbd942-0ce3-45e9-b1f9-f0af683db82e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b492aeef-172e-40ab-b849-ab2c633d9d13.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/87bde5bd-762b-4b47-ad13-9c1e4e083491.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cb526f64-6bc2-4960-bd71-d650e55123f7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/8b18255e-732e-425d-819b-a753a6191174.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6e45cfc3-cff9-4f47-ac1a-6d9525e2c883.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0463904e-94a1-4720-af81-08c5b289cbeb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/2e518748-1f82-41d1-8bd1-0628830456a7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ad79921b-4aac-4f54-bac9-59a624c95be9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/046e791a-e361-4383-8b6c-609fb1b6ca42.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b219aa06-e7bc-4649-ba96-03996a1565d9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d2e5e0da-7722-4f96-ad68-07e83bf3de5d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4ccfea7c-6c44-4f6f-a900-3aaa5354c356.jpg.710x400.jpg\"]", + "area": "浦东", + "town": "洋泾", + "price": "6.64w", + "priceDesc": "二手挂牌均价", + "plotType": "塔楼", + "leaseNum": "4", + "sellNum": "11", + "dealNum": "0", + "buildTime": "1993年建成", + "houseNum": "1344户", + "building": "6栋", + "propertyFee": "0.75元/平米/月", + "propertyCompany": "上海益镇物业管理有限公司", + "developers": "上海建工房产有限公司", + "lat": 31.247013798320733, + "lng": 121.54954132423184, + "imgArr": [ + "https://img.mapmapping.com/b9bb10e4-d216-4fe3-ac8d-0f3d778e288b.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/28582f65-a02a-47df-b730-9f7f5a5851ee.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/5997bbc0-2322-4d93-9714-2d4513db7677.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/4cd79c3a-f543-4a94-89e8-e80241ff7876.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/3b517fea-b3d3-4b1e-8f58-50e4775d7cfb.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/489d39ee-73ee-4d47-9f76-151c4313d5c3.png?imageView2/2/w/700/h/500/q/75" + ], + "focus": false, + "tags": [ + "浦东", + "洋泾", + "塔楼" + ], + "view": 15312, + "address": "上海市浦东新区栖山路401弄-11号-临" + }, + "timestamp": 1630576612732 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json new file mode 100644 index 000000000..9946d0550 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json @@ -0,0 +1,255 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "total": 4733, + "list": [ + { + "id": 59151, + "name": "凌高小区", + "area": "浦东", + "img": "https://img.mapmapping.com/a68e7f60-a8d8-4e25-ade8-2d77c86562cd.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "11", + "leaseNum": "4", + "price": "6.64w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.25306680262166000", + "lng": "121.55601323149715000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.53w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58324, + "name": "梅园四街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/775871a1-e5a9-4e01-88ed-a39f040b51ac.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "15", + "leaseNum": "11", + "price": "13.42w", + "formatPrice": null, + "year": "1988年建成", + "lat": "31.23754371649258600", + "lng": "121.53281691712684000", + "tags": [ + "近地铁2号线东昌路站" + ], + "plotType": "塔楼/板楼/塔板结合", + "buildTime": "1988年建成", + "view": "1.52w", + "tag": "近地铁2号线东昌路站" + }, + { + "id": 59165, + "name": "中星海上景庭", + "area": "浦东", + "img": "https://img.mapmapping.com/4f9d8c2a-8b89-49a8-b45f-ecaff4f3f588.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "8", + "leaseNum": "3", + "price": "10.86w", + "formatPrice": null, + "year": "2004年建成", + "lat": "31.24523893361267700", + "lng": "121.55501162085352000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "板楼", + "buildTime": "2004年建成", + "view": "1.50w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58477, + "name": "海怡别墅(公寓)", + "area": "浦东", + "img": "https://img.mapmapping.com/f50ed86d-474f-47c5-8f83-7c203c510d92.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "0", + "leaseNum": "0", + "price": "7.32w", + "formatPrice": null, + "year": "1994年建成", + "lat": "31.23963538757626600", + "lng": "121.53779361948267000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1994年建成", + "view": "1.50w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 59148, + "name": "崮山小区一街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/b651bbba-97ba-4ce0-831c-925e80b4459f.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范附属小学羽山校区", + "sellerNum": "10", + "leaseNum": "5", + "price": "11.81w", + "formatPrice": null, + "year": "1999年建成", + "lat": "31.24282189657667000", + "lng": "121.56992501943853000", + "tags": [ + "近地铁9号线芳甸路站" + ], + "plotType": "板楼", + "buildTime": "1999年建成", + "view": "1.50w", + "tag": "近地铁9号线芳甸路站" + }, + { + "id": 55895, + "name": "证大家园(四期)", + "area": "浦东", + "img": "https://img.mapmapping.com/d7d0671f-b0ed-4082-a9f6-1883eca3ac85.png?imageView2/2/w/240/h/200/q/75", + "town": "高行", + "schoolTag": "浦东新区福山证大外国语小学", + "sellerNum": "11", + "leaseNum": "7", + "price": "9.63w", + "formatPrice": null, + "year": "2006年建成", + "lat": "31.28831283871331500", + "lng": "121.60787797808233000", + "tags": [ + "近地铁6号线巨峰路站" + ], + "plotType": "板楼", + "buildTime": "2006年建成", + "view": "1.49w", + "tag": "近地铁6号线巨峰路站" + }, + { + "id": 58709, + "name": "新光大楼", + "area": "浦东", + "img": "https://img.mapmapping.com/f8e1dffb-fa85-457b-8a97-fb019239bdf6.png?imageView2/2/w/240/h/200/q/75", + "town": "潍坊", + "schoolTag": "浦东新区明珠小学A区", + "sellerNum": "3", + "leaseNum": "2", + "price": "11.95w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.23234474870607700", + "lng": "121.52462643697852000", + "tags": [ + "近地铁9号线商城路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.49w", + "tag": "近地铁9号线商城路站" + }, + { + "id": 59322, + "name": "盛世年华", + "area": "浦东", + "img": "https://img.mapmapping.com/83727155-29f5-47d9-b54d-4042d9e625ec.png?imageView2/2/w/240/h/200/q/75", + "town": "源深", + "schoolTag": "浦东新区福山花园外国语小学", + "sellerNum": "4", + "leaseNum": "11", + "price": "14.94w", + "formatPrice": null, + "year": "2005年建成", + "lat": "31.23494392135101000", + "lng": "121.54545194344688000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "2005年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58326, + "name": "梅园七街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/30cf630f-830b-4e43-8dc5-648bb7dfb760.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "2", + "leaseNum": "2", + "price": "11.13w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24378516838488700", + "lng": "121.53455423998314000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58599, + "name": "陈家门小区", + "area": "浦东", + "img": "https://img.mapmapping.com/7a9f43ef-67c4-4168-bf24-0190689d331d.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区梅园小学", + "sellerNum": "3", + "leaseNum": "1", + "price": "9.27w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24726096884606800", + "lng": "121.53135690118148000", + "tags": [ + "近地铁4号线浦东大道站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁4号线浦东大道站" + } + ], + "pageNum": 1, + "pageSize": 10, + "size": 10, + "startRow": 1, + "endRow": 10, + "pages": 474, + "prePage": 0, + "nextPage": 2, + "isFirstPage": true, + "isLastPage": false, + "hasPreviousPage": false, + "hasNextPage": true, + "navigatePages": 8, + "navigatepageNums": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ], + "navigateFirstPage": 1, + "navigateLastPage": 8 + }, + "timestamp": 1630575199849 +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json new file mode 100644 index 000000000..10af777c0 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json @@ -0,0 +1,38 @@ +{ + "code": 0, + "data": [ + { + "color": "74CBFF", + "name": "建行项目", + "groups": { + "name": "项目类别" + }, + "id": "3538604d04b1495585a45c23379b65f5" + }, + { + "color": "75ccff", + "name": "丁世菁", + "groups": { + "name": "商务负责人" + }, + "id": "c9055e8370464537a2eefc3aa925097b" + }, + { + "color": "88C896", + "name": "准备中", + "groups": { + "name": "项目进展" + }, + "id": "6138228645e347b3a1f6ed52ece457c5" + }, + { + "color": "75ccff", + "name": "董彦均", + "groups": { + "name": "商务负责人" + }, + "id": "8100d58e9a3e4c8cb9be9406147ee93b" + } + ], + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json new file mode 100644 index 000000000..d7510e331 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json @@ -0,0 +1,928 @@ +{ + "code": 0, + "data": { + "count": 176, + "list": [ + { + "code": "78", + "response": { + "name": "黄春英", + "userid": "143809622239903024" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "小区别名校验", + "planEndTime": "2021-05-14 23:59:00", + "id": "5c8f51d3971241eea4b1efc92b20fe5c", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 10:53:51" + }, + { + "code": "77", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-05-15 23:59:00", + "id": "3e12b0053aca4334ad28cd9bc30679f0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 10:19:16" + }, + { + "code": "76", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-05-15 23:59:00", + "id": "d6ccafed4b284951a5848ba62cfc00f0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:23:12" + }, + { + "code": "75", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "建行表整理及系统测试", + "planEndTime": "2021-05-15 23:59:00", + "id": "506a33123d984df98a05973fcc76f3d0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:12:20" + }, + { + "code": "74", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "系统测试", + "planEndTime": "2021-05-15 23:59:00", + "id": "bb3f9d174f9341b58b89203b3b326b9f", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:11:22" + }, + { + "code": "73", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-05-10 08:45:00", + "name": "导出小区别名数据给到人工检验,并整理人工反馈的小区别名校验结果;\n数据治理模型上线相关工作;", + "planEndTime": "2021-05-15 23:59:00", + "id": "ca1bae5186164b65b0807b597634317e", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 08:32:43" + }, + { + "code": "1.1", + "response": { + "name": "王辰岑", + "userid": "02245922215748" + }, + "planBeginTime": "2021-05-08 09:30:00", + "name": "网站注册及申请入驻供应商", + "planEndTime": "2021-05-08 18:00:00", + "id": "b1753f50c7a149f3a85c3a2fa0f180a8", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "7f01818d37c84c4d9337ef288623f5eb", + "abbrev": "A" + }, + "createDate": "2021-05-08 17:52:10" + }, + { + "code": "1", + "response": { + "name": "王辰岑", + "userid": "02245922215748" + }, + "name": "平台申请", + "id": "d357543551d14b22a91d2bd93b7afd31", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "7f01818d37c84c4d9337ef288623f5eb", + "abbrev": "A" + }, + "flow": { + "name": "蒋晓春", + "userid": "02245922395062" + }, + "createDate": "2021-05-08 17:49:05" + }, + { + "code": "11", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-05-08 09:00:00", + "name": "租金估值权限逻辑修改", + "planEndTime": "2021-05-08 19:00:00", + "id": "d0f6f2584e7440739c5b603f942c2bb7", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-05-08 09:42:10" + }, + { + "code": "27", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-06 09:00:00", + "name": "ES机器学习三台节点搭建", + "planEndTime": "2021-04-06 18:59:00", + "id": "a725c9be2bb2446eb180129fb58094d5", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:57:54" + }, + { + "code": "72", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "建行系统测试", + "planEndTime": "2021-05-08 23:59:00", + "id": "0c938c9b36054c84bc69ba166b31dea2", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:55:35" + }, + { + "code": "71", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "数据融合人工反馈处理", + "planEndTime": "2021-05-08 23:59:00", + "id": "daa8d8e8a69842eaae621a33e8690e55", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:43:55" + }, + { + "code": "26", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-05-07 09:00:00", + "name": "分析爬虫抓取情况,导出亲邻数据,导出四川锦城数据", + "planEndTime": "2021-05-07 18:00:00", + "id": "1c9085be784542fb917a2ff2767388c9", + "state": "待确认", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-05-07 15:39:42" + }, + { + "code": "70", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-07 00:00:00", + "name": "新增安居客小区详情解析,优化安居客小区详情抓取", + "id": "e7717e32c9864511aabfedd41117d6ca", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 14:37:30" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "微元素帐号整理", + "id": "4de48c77988743a5ab83343b40adee97", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "907da15cb4a54184bff9b32345300d2a", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:54:24" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 14:00:00", + "name": "公司名称及地址匹配", + "id": "286ad20909ca4b27bd70c34eeea6a3c1", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "270452cd5725475095c7e668ea137481", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:22:42" + }, + { + "code": "69", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 09:30:00", + "name": "新增搜房小区列表抓取及解析\n导出新租金米数据", + "planEndTime": "2021-05-06 14:00:00", + "id": "35df605842d941a58b94d6b5e3c8e5b7", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:21:21" + }, + { + "code": "68", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-05-08 23:59:00", + "id": "e3758204453b4858a02c33f7f0ceaf45", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 09:41:20" + }, + { + "code": "67", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-05-08 23:59:00", + "id": "613d42946b694b24b9c6d29a7fb3cbc8", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 09:41:00" + }, + { + "code": "66", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-05-06 08:45:00", + "name": "1.将小区别名数据导出给人工校验2.将78数据库中的人工标注数据,经纬度围栏数据放到85、88数据库", + "planEndTime": "2021-05-08 23:59:00", + "id": "2410d7840bff48af8b2202c7c7271cfb", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 08:30:36" + }, + { + "code": "10", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-05 00:00:00", + "name": "优化诸葛找房抓取稠租房少的问题\n新增搜房通过入小区页面抓取租房户房源\n及验证曹老师说抓取少的网站重抓", + "planEndTime": "2021-04-07 23:59:00", + "id": "c37324ae475149cfb0648731934873a9", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:35:09" + }, + { + "code": "25", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-29 00:00:00", + "name": "网站种子配置及福州二手房抓取", + "planEndTime": "2021-04-30 23:59:00", + "id": "a1326b626acf444193969c755287de1c", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:32:07" + }, + { + "code": "9", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-03 00:00:00", + "name": "新增我爱我家,百姓,赶集的抓取解析,\n新增我爱我家成交抓取,\n修改诸葛找房抓取,解析", + "planEndTime": "2021-04-03 23:59:00", + "id": "73e49abc26574e78b696113bcf200019", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:10:56" + }, + { + "code": "65", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "建行数据融合统计结果(彦均)", + "planEndTime": "2021-04-25 23:59:00", + "id": "be85282dc57d46e3a36fc0801107e605", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:03:21" + }, + { + "code": "3", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "建行数据导出(彦均)\n房源量小于20的小区\n小区统计", + "planEndTime": "2021-04-19 23:59:00", + "id": "af3c862c7c674f97855061cb923fbfef", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "ac8c4d70ea3842c2a6c628a64628989a", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:00:56" + }, + { + "code": "24", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-29 00:00:00", + "name": "通过小区抓安居客房源,配完,反爬厉害\n分析抓取现状\n总结输出房源方案", + "planEndTime": "2021-04-29 23:59:00", + "id": "bfccb3c436af4d1a9263ae4879dc7fee", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-29 18:29:05" + }, + { + "code": "1", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "企址库抓取\n尝试国家信息,天眼查,爱企查有js问题\n个企查查,启信宝有豋录问题,88不全,最后抓顺企", + "planEndTime": "2021-04-22 23:59:00", + "id": "b495c086d3fa4a6d8e6d7319fdef303f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "270452cd5725475095c7e668ea137481", + "abbrev": "A" + }, + "createDate": "2021-04-29 12:05:12" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-26 00:00:00", + "name": "建行案例测试批量评估价格,并并导出房源与神经网络j小区", + "planEndTime": "2021-04-28 23:59:00", + "id": "68afd0578fab469d8371fb510c1f8af9", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "ac8c4d70ea3842c2a6c628a64628989a", + "abbrev": "A" + }, + "createDate": "2021-04-29 11:51:14" + }, + { + "code": "26", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-26 00:00:00", + "name": "浙商本地化部署项目验证环境发布", + "planEndTime": "2021-04-28 23:59:00", + "id": "91ea49c8f5e747b18660dec02da7bfeb", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "b4abc1f7cad142c4be5e5b301ae08190", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-29 09:44:24" + }, + { + "code": "25", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-26 09:00:00", + "name": "浙商本地化验证环境部署工作", + "planEndTime": "2021-04-28 17:59:00", + "id": "09c28bed9a43475c9bd24a69066170d4", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "b4abc1f7cad142c4be5e5b301ae08190", + "abbrev": "A" + }, + "createDate": "2021-04-26 11:34:36" + }, + { + "code": "23", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "Python工具包封装", + "planEndTime": "2021-04-25 23:59:00", + "id": "f3a00c2e64184796a4a12ab37ce60ac5", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-25 18:24:34" + }, + { + "code": "64", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "系统测试,数据统计", + "planEndTime": "2021-04-30 23:59:00", + "id": "8b8be8a4a3424ac9a3cd74145d02f26f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 09:27:25" + }, + { + "code": "63", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-04-30 23:59:00", + "id": "1140039f97da40639763101258f4c300", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 09:14:40" + }, + { + "code": "62", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-30 23:59:00", + "id": "9c2064b98c814b3183189c6d6a68a068", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:41:04" + }, + { + "code": "61", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "96f4d6e9851b44fc84eb20a8fd58504a", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:39:14" + }, + { + "code": "60", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-04-25 08:45:00", + "name": "1.在82.83.84服务器上运行测试代码;\n2.编写”数据质量监测维度信息统计”的代码", + "planEndTime": "2021-04-30 23:59:00", + "id": "a354e6df53e34a9581fa5988f4334dfa", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:28:25" + }, + { + "code": "22", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "租金模型开发", + "planEndTime": "2021-04-27 23:59:00", + "id": "3f569aa260b04da88224834d56f43427", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-25 07:58:48" + }, + { + "code": "21", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "pyspider新增获取小区id接口及项目爬取时间监控", + "planEndTime": "2021-04-22 23:59:00", + "id": "f8122a43929646e8acfdbbf70091bf3c", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-21 09:44:31" + }, + { + "code": "59", + "response": { + "name": "黄春英", + "userid": "143809622239903024" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "别名表校验", + "planEndTime": "2021-04-25 23:59:00", + "id": "6a26dbc7534240f88861873177bca3ee", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-20 09:02:45" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "深圳房价角通室厅必填\n从室号解析楼层", + "planEndTime": "2021-04-19 18:00:00", + "id": "b4001b1b33fe486d93897d737dd950a3", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "cc6b1205b3d547f5867f44ddf5d568df", + "abbrev": "A" + }, + "createDate": "2021-04-19 19:23:25" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-13 00:00:00", + "name": "建行学校抓取合肥,苏苏州,西安", + "planEndTime": "2021-04-16 23:59:00", + "id": "e26d64fb288a4558bec894035db70f27", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "0fbd26a5a4914d92802352502819abd8", + "abbrev": "A" + }, + "createDate": "2021-04-19 19:14:47" + }, + { + "code": "58", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "建行数据系统开发", + "planEndTime": "2021-04-23 23:59:00", + "id": "de19c283d30843c9971d1eb5ba4aa157", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 16:14:04" + }, + { + "code": "20", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "修改均价测试系统bug", + "planEndTime": "2021-04-19 18:00:00", + "id": "d6f6e37425d24b2ba04ef1ce64958358", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-19 13:29:35" + }, + { + "code": "57", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "数据清洗", + "planEndTime": "2021-04-24 23:59:00", + "id": "a5d3e59e2fae4f42bc5097d50096edab", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:19:07" + }, + { + "code": "56", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "8ef405124f054f578fa4b21c7863c415", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:18:50" + }, + { + "code": "55", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "51d0a97ac69b48c89926ee7f2736e62d", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:06:04" + }, + { + "code": "54", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "楼盘治理上线联调", + "planEndTime": "2021-04-23 23:59:00", + "id": "d08c7c2c852048aebc8aa59d48cfbd21", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 08:43:11" + }, + { + "code": "53", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-04-19 08:45:00", + "name": "1.根据人工融合校验,优化清洗关键字和规则;2.在Linux的82.83.84服务器上测试代码", + "planEndTime": "2021-04-23 23:59:00", + "id": "2af0fe9397094718941cd081e5dfbe53", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 08:26:10" + }, + { + "code": "19", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "友达二手成交数据导入", + "planEndTime": "2021-04-19 18:00:00", + "id": "3a1891dfc95c4d39ac33ce64761a4a57", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-19 00:04:47" + }, + { + "code": "18", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-16 00:00:00", + "name": "深圳指导价入库与神经网络价格修复", + "planEndTime": "2021-04-16 23:59:00", + "id": "4999543a15ff429a85875348df048726", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-16 18:59:31" + } + ] + }, + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json new file mode 100644 index 000000000..f3e658cca --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json @@ -0,0 +1,225 @@ +{ + "code": 0, + "data": { + "count": 18, + "list": [ + { + "taskCount": 2, + "taskendCount": 2, + "name": "建行配套数据采购项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "77478f60337840ccaf5ebb294ac1c987", + "state": "进行中", + "createDate": "2021-04-12 17:51:21" + }, + { + "taskCount": 3, + "taskendCount": 3, + "name": "房金部房产估值模型", + "createUser": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "id": "e7caf92da7be4c529c8733cbb196173e", + "state": "进行中", + "createDate": "2021-04-12 08:43:54" + }, + { + "taskCount": 1, + "taskendCount": 0, + "name": "地址标准化项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f7c307d9af794f4589f0a7027142e6e8", + "state": "进行中", + "createDate": "2021-04-01 10:03:56" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "企业经营地址库", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "59b1247428a445f5971f8f06091fb558", + "state": "进行中", + "createDate": "2021-03-22 15:10:41" + }, + { + "taskCount": 11, + "taskendCount": 8, + "name": "建信住房租金估价报告项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "ecd77309ab1c4b83b54ee87b8ddd4f98", + "state": "进行中", + "createDate": "2021-03-19 10:12:40" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "上数分地址标准化项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "4f22540647c74fc0a8277d798fbe9bfc", + "state": "进行中", + "createDate": "2021-03-18 16:50:19" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "建行开发商项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f82329c2390c4d2399f96ba21e92c950", + "state": "进行中", + "createDate": "2021-03-15 09:52:34" + }, + { + "taskCount": 9, + "taskendCount": 8, + "name": "房金部楼盘字典数据测试", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "457f11e89aad42b6a7bb289e8341ac04", + "state": "进行中", + "createDate": "2021-03-12 17:40:02" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "房价通", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "6a218757c86648559dacd9e4538d3f0a", + "state": "进行中", + "createDate": "2021-03-11 14:49:37" + }, + { + "taskCount": 1, + "taskendCount": 1, + "name": "亲邻科技", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "33c2a17a6d194168a2838c9ae4bd18b3", + "state": "进行中", + "createDate": "2021-03-10 11:46:36" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "福州农商银行", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f1f7bf085c3b4033932b8e78d2a644ed", + "state": "进行中", + "createDate": "2021-03-10 10:43:23" + }, + { + "taskCount": 27, + "taskendCount": 25, + "name": "房价网基础数据建设", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "ddd31486abb645eaa8a48fb43d57045b", + "state": "进行中", + "createDate": "2021-03-10 10:27:53" + }, + { + "taskCount": 1, + "taskendCount": 1, + "name": "江南农村商业银行", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "8f18fddec76a4ca788946825a741abdc", + "state": "进行中", + "createDate": "2021-03-09 09:38:15" + }, + { + "taskCount": 72, + "taskendCount": 65, + "name": "建行楼盘数据治理项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "41bdf23cbf634035b694f92ba72b067f", + "state": "进行中", + "createDate": "2021-03-02 16:54:38" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "江苏农行项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "a10e63ac24b9487cb1dd1140dd2ffc94", + "state": "进行中", + "createDate": "2021-03-02 09:08:45" + }, + { + "taskCount": 0, + "taskendCount": 0, + "name": "浙商银行估值项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "53aa253f216542dfb8022dcbd18601cc", + "state": "进行中", + "createDate": "2021-03-02 09:07:54" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "机房运维", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "fc76aa65155741c1a074305741a067eb", + "state": "进行中", + "createDate": "2021-03-02 09:01:42" + }, + { + "taskCount": 27, + "taskendCount": 26, + "name": "浙商银行本地化部署", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "71e520aff8e243f3b04e9d6fea7867d1", + "state": "进行中", + "createDate": "2021-03-01 16:36:41" + } + ] + }, + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json new file mode 100644 index 000000000..125089046 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json @@ -0,0 +1,65 @@ +{ + "code": "000", + "desc": "操作成功", + "content": [ + { + "id": 272113, + "name": "第六师范学校第二附属小学西校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "小学", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "41", + "sellNum": "188", + "plotPrice": "8.69", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + }, + { + "id": 273287, + "name": "进才中学北校苗圃路校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "初中", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "85", + "sellNum": "339", + "plotPrice": "9.33", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + } + ], + "timestamp": 1630576702404 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json new file mode 100644 index 000000000..8026c9d69 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json @@ -0,0 +1,85 @@ +{ + "code": 0, + "data": { + "count": 4, + "list": [ + { + "code": "5", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-06 09:00:00", + "name": "地图展示上海租金数据", + "planEndTime": "2021-04-06 18:00:00", + "id": "3bbb043ad0b44ea2a1213e725274e3ad", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-06 15:52:01" + }, + { + "code": "6", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-07 09:00:00", + "name": "租房报告预览下载;es迁移丢失数据同步", + "planEndTime": "2021-04-08 18:59:00", + "id": "8725262c17db408dab1f31473d50598f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-07 09:31:53" + }, + { + "code": "7", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-08 09:00:00", + "name": "根据新的租房报告进仁行接口修改", + "planEndTime": "2021-04-08 18:00:00", + "id": "309bdc4420e74868bda27884f4a46508", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-12 11:24:01" + }, + { + "code": "8", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-16 09:23:00", + "name": "估值页面报告优化及小学数据按标准导出", + "planEndTime": "2021-04-16 19:00:00", + "id": "45f9d101b18b48a08045de76cf2e53af", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-16 09:23:21" + } + ] + }, + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json new file mode 100644 index 000000000..362cb7398 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json @@ -0,0 +1,29 @@ +{ + "code": 0, + "data": { + "code": "5", + "realityBeginDate": "2021-04-06 09:00:00", + "assignDate": "2021-04-06 15:52:01", + "flowConten": "1", + "description": "", + "realityEndDate": "2021-04-08 16:49:07", + "workTime": "1.00", + "confirmDate": "2021-04-08 16:49:07", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "name": "地图展示上海租金数据", + "inventoryId": "d00fcb9c5307499eab8f9798a23f6360", + "state": "已完成", + "id": "3bbb043ad0b44ea2a1213e725274e3ad", + "planEndDate": "2021-04-06 18:00:00", + "planBeginDate": "2021-04-06 09:00:00", + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-06 15:52:01" + }, + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java new file mode 100644 index 000000000..8192b359c --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoiDetail.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2022/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoiDetail { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java new file mode 100644 index 000000000..b02953be7 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java @@ -0,0 +1,79 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseDetail { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "imgsUrl") + private String imgsUrl; + @JSONField(name = "area") + private String area; + @JSONField(name = "town") + private String town; + @JSONField(name = "price") + private String price; + @JSONField(name = "priceDesc") + private String priceDesc; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "dealNum") + private String dealNum; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "houseNum") + private String houseNum; + @JSONField(name = "building") + private String building; + @JSONField(name = "propertyFee") + private String propertyFee; + @JSONField(name = "propertyCompany") + private String propertyCompany; + @JSONField(name = "developers") + private String developers; + @JSONField(name = "lat") + private Double lat; + @JSONField(name = "lng") + private Double lng; + @JSONField(name = "imgArr") + private List imgArr; + @JSONField(name = "focus") + private Boolean focus; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "view") + private Integer view; + @JSONField(name = "address") + private String address; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java new file mode 100644 index 000000000..cd49ae026 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java @@ -0,0 +1,107 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseList { + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "total") + private Integer total; + @JSONField(name = "list") + private List list; + @JSONField(name = "pageNum") + private Integer pageNum; + @JSONField(name = "pageSize") + private Integer pageSize; + @JSONField(name = "size") + private Integer size; + @JSONField(name = "startRow") + private Integer startRow; + @JSONField(name = "endRow") + private Integer endRow; + @JSONField(name = "pages") + private Integer pages; + @JSONField(name = "prePage") + private Integer prePage; + @JSONField(name = "nextPage") + private Integer nextPage; + @JSONField(name = "isFirstPage") + private Boolean isFirstPage; + @JSONField(name = "isLastPage") + private Boolean isLastPage; + @JSONField(name = "hasPreviousPage") + private Boolean hasPreviousPage; + @JSONField(name = "hasNextPage") + private Boolean hasNextPage; + @JSONField(name = "navigatePages") + private Integer navigatePages; + @JSONField(name = "navigatepageNums") + private List navigatepageNums; + @JSONField(name = "navigateFirstPage") + private Integer navigateFirstPage; + @JSONField(name = "navigateLastPage") + private Integer navigateLastPage; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "area") + private String area; + @JSONField(name = "img") + private String img; + @JSONField(name = "town") + private String town; + @JSONField(name = "schoolTag") + private String schoolTag; + @JSONField(name = "sellerNum") + private String sellerNum; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "price") + private String price; + @JSONField(name = "formatPrice") + private Object formatPrice; + @JSONField(name = "year") + private String year; + @JSONField(name = "lat") + private String lat; + @JSONField(name = "lng") + private String lng; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "view") + private String view; + @JSONField(name = "tag") + private String tag; + } + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java new file mode 100644 index 000000000..992727e92 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java @@ -0,0 +1,66 @@ +package com.fangjia.script.statistics.seaweed; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class School { + + + @JsonProperty("code") + private String code; + @JsonProperty("desc") + private String desc; + @JsonProperty("content") + private ContentDTO content; + @JsonProperty("timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JsonProperty("total") + private Integer total; + @JsonProperty("hasNextPage") + private Boolean hasNextPage; + @JsonProperty("list") + private List list; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JsonProperty("id") + private Integer id; + @JsonProperty("name") + private String name; + @JsonProperty("area") + private String area; + @JsonProperty("shortName") + private Object shortName; + @JsonProperty("picture") + private String picture; + @JsonProperty("distance") + private String distance; + @JsonProperty("plotPrice") + private String plotPrice; + @JsonProperty("topInfo") + private Object topInfo; + @JsonProperty("view") + private String view; + @JsonProperty("town") + private String town; + @JsonProperty("address") + private String address; + @JsonProperty("tags") + private List tags; + } + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 46545fc04..15909a4c9 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,12 +1,12 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; -import com.fangjia.core.util.*; +import com.fangjia.core.util.DataUtil; +import com.fangjia.core.util.GeoUtils; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; -import com.fangjia.dal.mongo.model.AmapPoi; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.seaweed.dao.SeaweedDao; @@ -16,17 +16,15 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors; import com.fangjia.service.address.processor.AddressProcessors; import com.fangjia.service.api.AliyunService; import com.fangjia.service.bo.AliyunBo; -import com.fangjia.service.bo.MatchAddressScore; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.addrepo.AddressRepoService; import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.LoudongService; import com.google.common.base.Function; import com.google.common.collect.MapMaker; -import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import org.bson.types.ObjectId; + import java.util.*; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; @@ -759,6 +757,13 @@ public class AddressParserServiceImpl implements AddressParserService { double simScore; boolean onSameRoad = false; boolean sameDistrict = isSameDistrict(address, metaData); + if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){ + return 0; + } +// //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区 +// if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){ +// return 0; +// } Iterator> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator(); Map.Entry addrEntry; while(addrEntrySet.hasNext()){ @@ -808,7 +813,6 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - String keywordPinYin = address.getKeywordsPinYin(); String keywords = address.getKeywords(); String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin(); @@ -892,7 +896,7 @@ public class AddressParserServiceImpl implements AddressParserService { if(name.length() == 2 && (name.equals(address.getOrgAddress()) || name.equals(address.getEffectiveAddress()))){ keywordScore = Math.max(keywordScore, namePriority); }else if(name.length() > 3 && (address.getOrgAddrePinYin().contains(namePinYin) || enhanceContains(address.getOrgAddress(), name))){ - double similar = AddressParserUtil.getSimilarityRatio(name, address.getEffectiveAddress()); + double similar = AddressParserUtil.getSimilarityRatio(name, address.getOrgAddress()); if(similar == 1){ keywordScore = Math.max(similar, keywordScore) * namePriority * 1.0001; }else if(similar > 0.1){ @@ -907,20 +911,27 @@ public class AddressParserServiceImpl implements AddressParserService { if (standardAddressScore > 0 && keywordScore > 0) { matchLevelScore = 0.9; similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE; - } else if(keywordScore > 0 && streetScore > 0){ - matchLevelScore = 0.8; - similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; } else if (standardAddressScore > 0) { - matchLevelScore = 0.7; + matchLevelScore = 0.8; similarityScore = standardAddressScore * SIMILARITY_RATE; + } else if(keywordScore > 0 && streetScore > 0){ + matchLevelScore = 0.7; + similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; } else if (keywordScore > 0) { matchLevelScore = 0.6; similarityScore = keywordScore * SIMILARITY_RATE; } else if (fuzzyKeywordsScore > 0) { - matchLevelScore = 0.5; + if(address.hasCoordinate() && metaData.getLng() != null && GeoUtils.calculateDistance(address.getLng(), address.getLat(), + metaData.getLng(), metaData.getLat()) < 100){ + matchLevelScore = 0.7; + }else{ + matchLevelScore = 0.4; + } similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE; + }else if(metaData.getId().equals(address.getPolygonSeaweedId())){ + matchLevelScore = 0.4; + similarityScore = 0.05; } - if (similarityScore > 0) { if(metaData.getBuildingType() != null){ if ("shop".equals(metaData.getBuildingType().toString())) { @@ -942,10 +953,6 @@ public class AddressParserServiceImpl implements AddressParserService { * similarityScore = 相似度部分(0.07) + 类型部分(0.01) < 0.1 * */ double score = matchLevelScore + similarityScore; - if(address.hasCoordinate() && metaData.hasCoordinate() - && GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) > 3 * 1000){ - score = 0d; - } return score; } @@ -980,7 +987,7 @@ public class AddressParserServiceImpl implements AddressParserService { return false; } if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){ - return false; + return true; } String district = metaData.getDistrict(); return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 1c8f2d004..ed723c1d5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -61,6 +61,8 @@ public class Address { private String suggestDistrict; private String suggestName; private String suggestNamePinYin; + private String polygonSeaweedId; + private String geoRegion; private List buildingNumbers; private PreciseLevel preciseLevel; private boolean pinYin; @@ -397,6 +399,7 @@ public class Address { } public void setOrgAddress(String orgAddress) { + orgAddress = orgAddress.toUpperCase(); setAddress(orgAddress); if(this.orgAddress != null){ throw new UnsupportedOperationException("原始地址不能被修改"); @@ -749,6 +752,14 @@ public class Address { return false; } + public String getGeoRegion() { + return geoRegion; + } + + public void setGeoRegion(String geoRegion) { + this.geoRegion = geoRegion; + } + @Override public String toString() { return "Address{" + @@ -808,6 +819,14 @@ public class Address { '}'; } + public String getPolygonSeaweedId() { + return polygonSeaweedId; + } + + public void setPolygonSeaweedId(String polygonSeaweedId) { + this.polygonSeaweedId = polygonSeaweedId; + } + public String toMd5Hex(){ String keyProperties = "Address{" + "city='" + city + '\'' + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java index aa72e6d01..d1a15d156 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java @@ -20,18 +20,16 @@ public class AddressProcessors{ static { //修改顺序需要调整process i 的值 - addressProcessors.add(new TokenProcessor()); addressProcessors.add(new ProximalCharactersProcessor()); addressProcessors.add(new PreNoiseProcessor()); addressProcessors.add(new AddressFormatProcessor()); - addressProcessors.add(new TokenProcessor()); addressProcessors.add(new FloorProcessor()); addressProcessors.add(new TooltipProcessor()); addressProcessors.add(new ProvinceProcessor()); addressProcessors.add(new CityProcessor()); addressProcessors.add(new DistrictProcessor()); - addressProcessors.add(new LoopLineProcessor()); addressProcessors.add(new TownshipProcessor()); + addressProcessors.add(new LoopLineProcessor()); addressProcessors.add(new StageAndPartitionProcessor()); addressProcessors.add(new DirectionProcessor()); addressProcessors.add(new OffsetProcessor()); @@ -44,7 +42,7 @@ public class AddressProcessors{ addressProcessors.add(new NumericProcessor()); addressProcessors.add(new ProvinceAdjustProcessor()); addressProcessors.add(new FormatProcessor()); -// addressProcessors.add(new GeoProcessor()); + addressProcessors.add(new GeoProcessor()); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 6f88c20ef..fce876ec2 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -52,8 +52,9 @@ public class CityProcessor extends AbstractAddressProcessor { } if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){ String formatCity = RegionData.formatCity(city); - String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null; - String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity; + address.setCity(formatCity); + String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null; + String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity; /**如果省份和城市信息相互矛盾 则用区域推断城市*/ if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){ List possibleCities = speculateCityNameByRegion(address, city); @@ -191,15 +192,12 @@ public class CityProcessor extends AbstractAddressProcessor { }else{ weight.score += 2; } - }else if(address.contains(city)){ index = address.indexOf(city); weight.index = index; index += city.length(); if(province != null && (orgAddress.contains(province + city) || orgAddress.contains(ProvinceData.getDisplay(province) + city))){ weight.score += 2; - }else{ - weight.score += 1; } } Map> regionMap = RegionData.getAllRegions(city); @@ -208,9 +206,11 @@ public class CityProcessor extends AbstractAddressProcessor { if(matchResult !=null){ address = address.replace(matchResult.matchContent, ""); } - if(address.indexOf(region, index) > -1){ + if(address.indexOf(region + "区", index) > -1 || address.indexOf(region + "县", index) > -1){ if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){ weight.score += 2; + }else if(city.equals(addr.getCity())){ + weight.score += 10; }else{ weight.score += 1; } @@ -221,9 +221,13 @@ public class CityProcessor extends AbstractAddressProcessor { List alias = regionMap.get(region); if(alias != null){ for(String alia : alias){ - if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1 + if(StringUtils.isNotBlank(alia) && !alia.equals(region) && alia.length() > 2 && address.indexOf(alia, index) > -1 && (alia.endsWith("市")|(alia.endsWith("县")))){ - weight.score += 1; + if(city.equals(addr.getCity())){ + weight.score += 10; + }else{ + weight.score += 1; + } weight.index = address.indexOf(alia, index); break; } @@ -237,16 +241,6 @@ public class CityProcessor extends AbstractAddressProcessor { } List weights = new ArrayList(weightMap.values()); if(weights.size() > 0){ - /*String street = getStreet(address); - if(street != null){ - Iterator iterator = weights.iterator(); - while(iterator.hasNext()){ - FragmentWeight fragmentWeight = iterator.next(); - if(match(street, ".*" + fragmentWeight.text + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)")){ - iterator.remove(); - } - } - }*/ Collections.sort(weights); } if(weights.size() > 0){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java index 5cfba96b5..697e69f8d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java @@ -167,6 +167,9 @@ public class DistrictProcessor extends AbstractAddressProcessor { address.setDevelopmentZone(dz); } } + if(address.getSuggestDistrict() != null && address.getDistrict() == null){ + address.setDistrict(address.getSuggestDistrict()); + } address.setAddress(orgAddress); address.setEffectiveAddress(address.getAddress()); return RepeatStatus.NEXT; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index 83d74a0d4..e3ef9a3da 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -5,8 +5,12 @@ import com.fangjia.core.util.HttpRequest; import com.fangjia.service.address.mode.Address; import net.sf.json.JSONArray; import net.sf.json.JSONObject; -import org.apache.commons.lang.StringUtils; -import java.util.*; +import org.apache.log4j.Logger; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * 地址经纬度处理器 @@ -16,78 +20,69 @@ import java.util.*; public class GeoProcessor extends AbstractAddressProcessor { private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde"; - - List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); - List DISTRICT_LEVEL = Arrays.asList("区县", "开发区", "乡镇", "村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); + private static final Logger logger = Logger.getLogger(GeoProcessor.class); + List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口", "门牌号"); @Override public RepeatStatus process(Address address) { - if (address.hasCoordinate()) { return RepeatStatus.NEXT; } - List queryTypes = new ArrayList(); - List addressList = new ArrayList(); - if (address.isStandardAddress()) { - addressList.add(address.getStandardAddress()); - queryTypes.add("StandardAddress"); - } - if (StringUtils.isNotBlank(address.getOrgKeywords())) { - addressList.add(address.getOrgKeywords()); - queryTypes.add("Keywords"); + String addr = address.getEffectiveAddress(); + if(addr == null || addr.length() < 3){ + addr = address.getOrgAddress(); } HttpRequest request = new HttpRequest(); Map params = new HashMap(); - params.put("address", StringUtils.join(addressList, "|")); + params.put("address", addr); params.put("key", KEY); params.put("city", address.getCity()); - params.put("batch", "true"); params.put("output", "json"); try { JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8"); + logger.info("高德API:" + response.toString()); if (response.getInt("status") == 1) { JSONArray geocodes = response.getJSONArray("geocodes"); - if(queryTypes.contains("StandardAddress")){ - JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("StandardAddress")); - String level = geocode.getString("level"); - if(DISTRICT_LEVEL.contains(level)){ - if(StringUtils.isBlank(address.getDistrict())){ - address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); + if(geocodes.size() > 0){ + JSONObject json = geocodes.getJSONObject(0); + if(json.get("district") instanceof String){ + String city = address.getCity(); + if(city == null){ + city = RegionData.formatCity(json.getString("city")); } + String region = RegionData.formatRegion(city, json.getString("district")); + address.setGeoRegion(region); } - if(geocode.containsKey("street") && geocode.containsKey("number")){ - String geocodeAddress = geocode.getString("street") + geocode.getString("number"); - if(address.getStandardAddress().contains(geocodeAddress)){ - String location = geocode.getString("location"); - address.setLng(Double.parseDouble(location.split(",")[0])); - address.setLat(Double.parseDouble(location.split(",")[1])); - } - } - } - if(queryTypes.contains("Keywords")){ - JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("Keywords")); - String level = geocode.getString("level"); - if(DISTRICT_LEVEL.contains(level)){ - if(StringUtils.isBlank(address.getDistrict())){ - address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); - } - } - if(POI_LEVELS.contains(level)){ - String formattedAddress = geocode.getString("formatted_address"); - formattedAddress = formattedAddress - .replaceFirst(geocode.getString("province"), "") - .replaceFirst(geocode.getString("city"), "") - .replaceFirst(geocode.getString("district"), ""); - if(formattedAddress.equals(address.getOrgKeywords())){ - String location = geocode.getString("location"); - address.setLng(Double.parseDouble(location.split(",")[0])); - address.setLat(Double.parseDouble(location.split(",")[1])); + if(POI_LEVELS.contains(json.getString("level"))){ + //对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼 + if("门牌号".equals(json.getString("level"))){ + if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){ + return RepeatStatus.NEXT; + } } + String location = json.getString("location"); + address.setLng(Double.valueOf(location.split(",")[0])); + address.setLat(Double.valueOf(location.split(",")[1])); + String tmp = "{ \"_source\": [\"seaweed_id\"], \"query\": {\"geo_shape\": {\"shape\": {\"shape\": {\"type\": \"Point\",\"coordinates\":[location]},\"relation\": \"CONTAINS\"}}}}"; + tmp = tmp.replace("location", location); + Map headers = new HashMap(); + headers.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz"); + JSONObject resp = JSONObject.fromObject(request.postWithBody("http://192.168.0.206:9200/seaweed_polygon/_search", JSONObject.fromObject(tmp),headers,"UTF-8")); + if(resp.containsKey("hits") && resp.getJSONObject("hits").containsKey("hits")){ + JSONArray hits = resp.getJSONObject("hits").getJSONArray("hits"); + if(hits.size() > 0){ + String seaweedId = hits.getJSONObject(0).getJSONObject("_source").getString("seaweed_id"); + address.setPolygonSeaweedId(seaweedId); + } + } + } } + } } catch (Exception e) { - + logger.error(e); + e.printStackTrace(); } return RepeatStatus.NEXT; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java index 547d8d2cb..b982d0b4f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/KeywordProcessor.java @@ -22,7 +22,7 @@ public class KeywordProcessor extends AbstractAddressProcessor { static final List KEYWORDS_PATTERNS = Arrays.asList(Pattern.compile(".*[^一二三四五六七八九十\\d]{3,}号")); static final List KEYWORDS = Arrays.asList("居委会", "辖区", "街道办", "街道", "办事处", "村委", "管理区"); static final List NOISE_PATTERNS = Arrays.asList(Pattern.compile("^.*\\d+队"), Pattern.compile("^\\d+号院$")); - static final List INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属院")); + static final List INVALID_KEYWORD_PATTERN = Arrays.asList(Pattern.compile("^[一二三四五六七八九十甲乙丙丁戊己庚辛A-Z\\d]+[区期]$"), Pattern.compile("碧桂园"), Pattern.compile("住宅楼"), Pattern.compile("家属[区院]"), Pattern.compile("宿舍"), Pattern.compile("生活区"), Pattern.compile("公寓")); static final Pattern INVALID_PATTERN = Pattern.compile("^([-\\\\、])+$"); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java index ab5c929b3..7e4be0d63 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java @@ -23,11 +23,17 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor { addr = addr.replaceAll(matchResult.matchContent, formatStage); address.setAddress(addr); } - address.setStage(matchResult.matchContent); + address.setStage(formatStage); } matchResult = matchFirst(AddressParserPatterns.PARTITION_PATTERN, orgAddr); if (matchResult != null) { - address.setPartition(matchResult.matchContent); + String formatPartition = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent); + if(!formatPartition.equals(matchResult.matchContent)){ + String addr = address.getAddress(); + addr = addr.replaceAll(matchResult.matchContent, formatPartition); + address.setAddress(addr); + } + address.setStage(formatPartition); } return RepeatStatus.NEXT; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java index 4d3b65b14..3921fddf5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/TooltipProcessor.java @@ -42,12 +42,10 @@ public class TooltipProcessor extends AbstractAddressProcessor { if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){ address.setTooltip(tooltip); address.setStreet(getStreet(tooltip)); - } else if(containsDistrictName(tooltip)){ - address.setTooltip(tooltip); - } else if(RegionData.isRegion(address.getCity(), tooltip)){ + }else if(RegionData.isRegion(address.getCity(), tooltip)){ address.setSuggestDistrict(tooltip); }else{ - address.addNosie(tooltip); + address.setTooltip(tooltip); } } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java index 5236b7d55..797d9e367 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java @@ -15,15 +15,6 @@ public class VillageNameProcessor extends AbstractAddressProcessor { @Override public AbstractAddressProcessor.RepeatStatus process(Address address) { String orgAddress = address.getAddress(); - if(address.getTokens()!=null){ - for(Token token:address.getTokens()){ - if("district".equals(token.getKeywordTypes())){ - address.setKeywords(token.getName()); - address.setAddress(orgAddress.replace(token.getName(),"")); - return RepeatStatus.NEXT; - } - } - } List names = tokenDistrictNames(orgAddress); List kws = new ArrayList(); for(String name : names){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java index de010e201..c216a1eba 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java @@ -1,24 +1,16 @@ package com.fangjia.service.cnAnalyzer; -import java.io.IOException; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fangjia.core.util.DataUtil; import com.goojia.solr.ganalytics.IKSegmentation; import com.goojia.solr.ganalytics.Lexeme; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.io.UnsupportedEncodingException; +import java.util.*; /** * 房价网分词器 diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java new file mode 100644 index 000000000..733fcb368 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationService.java @@ -0,0 +1,11 @@ +package com.fangjia.service.datastore; + +import com.fangjia.dal.mongo.model.BsEvaluation; + +import java.util.Date; +import java.util.List; + +public interface BsEvaluationService { + + List getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java new file mode 100644 index 000000000..4d96aaadd --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/BsEvaluationServiceImpl.java @@ -0,0 +1,44 @@ +package com.fangjia.service.datastore; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.SortParam; +import com.fangjia.dal.mongo.dao.BsEvaluationDao; +import com.fangjia.dal.mongo.dao.EvaluationDao; +import com.fangjia.dal.mongo.model.BsEvaluation; +import com.fangjia.dal.mongo.model.bkoff.DealPrice; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +public class BsEvaluationServiceImpl implements BsEvaluationService { + + private BsEvaluationDao bsEvaluationDao; + + public void setBsEvaluationDao(BsEvaluationDao bsEvaluationDao) { + this.bsEvaluationDao = bsEvaluationDao; + } + + @Override + public List getNearlySameEvaluation(String id, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, Integer limit) { + List params = new ArrayList(); + params.add(new QueryParam("commid", id)); + if(startDate!=null){ + params.add(new QueryParam("cDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate)); + } + if(endDate!=null){ + params.add(new QueryParam("cDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate)); + } + + if (minPrice > 0) { + params.add(new QueryParam("unitPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice)); + } + if (maxPrice > 0) { + params.add(new QueryParam("unitPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice)); + } + List sortParams = new ArrayList(); + sortParams.add(new SortParam("cDate", false)); + List bsEvaluations = bsEvaluationDao.findAllByParams(params, sortParams, limit); + return bsEvaluations; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java index bea9e7589..543a2f6ce 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/datastore/DistrictServiceImpl.java @@ -2014,7 +2014,13 @@ public class DistrictServiceImpl implements DistrictService { params.add(new QueryParam("city", ClientUtils.escapeQueryChars(city))); } if (StringUtils.isNotBlank(region)) { - params.add(new QueryParam("region", ClientUtils.escapeQueryChars(RegionData.formatRegion(city, region)))); + city = ClientUtils.escapeQueryChars(city); + String newRegion = RegionData.formatRegion(city, region); + if(StringUtils.isNotBlank(newRegion)){ + params.add(new QueryParam("region", ClientUtils.escapeQueryChars(newRegion))); + }else{ + params.add(new QueryParam("region", region)); + } } if (StringUtils.isNotBlank(target)) { params.add(new QueryParam("category", target)); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java index 3965c9f00..c38f74a88 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java @@ -240,7 +240,7 @@ public class AddressRepoServiceImpl implements AddressRepoService { } private boolean isAvailablePoi(AddressMatchMetaData data){ - if(data.getType() != null && data.getType().contains("路")){ + if(data.getType() != null && (data.getType().contains("路") || data.getType().contains("街道"))){ return false; } return true; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java index 3f6ad7f39..236cf00c0 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongService.java @@ -94,4 +94,6 @@ public interface HouseLoudongService { public List findAll(List queryParams, Pager pager); List findAllSources(); + + public List findAll(String id, String houseNum, String unitNum, String roomNum); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java index 70286758b..9fe6af3ef 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/HouseLoudongServiceImpl.java @@ -297,6 +297,22 @@ public class HouseLoudongServiceImpl implements HouseLoudongService { return froms; } + @Override + public List findAll(String id, String houseNum, String unitNum, String roomNum) { + List params = new ArrayList(); + params.add(new QueryParam("districtId", id)); + if (StringUtils.isNotBlank(houseNum)) { + params.add(new QueryParam("houseNum", houseNum)); + } + if (StringUtils.isNotBlank(unitNum)) { + params.add(new QueryParam("houseNumUnit", unitNum)); + } + if (StringUtils.isNotBlank(roomNum)) { + params.add(new QueryParam("roomNum", roomNum)); + } + return houseLoudongBkoffDao.findAllByParams(params); + } + @Override public List queryHouseLoudongList(String city, String districtName, String roomNum, Pager pager) { List params = new ArrayList(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java index a38fdab95..d6dc0d63b 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongService.java @@ -23,6 +23,8 @@ public interface LoudongService { List findAll(String city, String region, String name, String houseNumber, String unitNumber, int limit); + List findAll(String id, String houseNumber, String unitNumber); + Map findAll(List query, String city); Loudong findById(String id); @@ -41,4 +43,6 @@ public interface LoudongService { void save(Map data); List findAllSources(); + + Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java index 57e9c6bad..d90620ac3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/LoudongServiceImpl.java @@ -148,6 +148,23 @@ public class LoudongServiceImpl implements LoudongService { return loudongDao.findAllByParams(params,null, 0 , limit, ADDRESS_VERIFY_FIELDS); } + @Override + public List findAll(String id, String houseNumber, String unitNumber) { + List params = new ArrayList(); + if(StringUtils.isNotBlank(id)) { + params.add(new QueryParam("idInSeaweed", id)); + } + + if(StringUtils.isNotBlank(houseNumber)) { + params.add(new QueryParam("houseNumber", houseNumber)); + } + if(StringUtils.isNotBlank(unitNumber)) { + params.add(new QueryParam("unitNumber", unitNumber)); + } + + return loudongDao.findAllByParams(params); + } + @Override public Loudong findById(String id) { return loudongDao.findById(id); @@ -318,4 +335,23 @@ public class LoudongServiceImpl implements LoudongService { } return froms; } + + @Override + public Loudong findForEvaluate(String idInSeaweed, String houseNumber, String unitNumber){ + if(StringUtils.isBlank(idInSeaweed) || StringUtils.isBlank(houseNumber)){ + return null; + } + List params = Lists.newArrayList(); + params.add(new QueryParam("idInSeaweed", idInSeaweed)); + params.add(new QueryParam("houseNumber", houseNumber)); + if(StringUtils.isNotBlank(unitNumber)) { + params.add(new QueryParam("unitNumber", unitNumber)); + } + params.add(new QueryParam("floors", QueryParam.GREATER_THAN_OPERATOR, 0)); + params.add(new QueryParam("floors", QueryParam.LESS_THAN_OPERATOR, 60)); + params.add(new QueryParam("status", QueryParam.NOT_EQUAL_OPERATOR, 0)); + + List sortParams = ImmutableList.of(new SortParam("floors", false)); + return loudongDao.findFirstByParams(params, sortParams); + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java index 9061a3288..fbd4dd7ea 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceService.java @@ -82,6 +82,8 @@ public interface DealPriceService { public List getNearlySameDealPrice(DistrictKey key, int minPrice, int maxPrice, int radius, int limit); + public List getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice,Date startDate, Date endDate, int limit); + public List getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit); /** diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java index be0f7261f..dd205bd77 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/fangjia/DealPriceServiceImpl.java @@ -354,6 +354,29 @@ public class DealPriceServiceImpl implements DealPriceService { return houses; } + @Override + public List getNearlySameDealPrice(String districtId, Integer minPrice, Integer maxPrice, Date startDate, Date endDate, int limit) { + List params = new ArrayList(); + params.add(new QueryParam("matchId", districtId)); + if(startDate!=null){ + params.add(new QueryParam("tradeDate", QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate)); + } + if(endDate!=null){ + params.add(new QueryParam("tradeDate", QueryParam.LESS_THAN_EQUAL_OPERATOR, startDate)); + } + + if (minPrice > 0) { + params.add(new QueryParam("avgPrice", QueryParam.GREATER_THAN_OPERATOR, minPrice)); + } + if (maxPrice > 0) { + params.add(new QueryParam("avgPrice", QueryParam.LESS_THAN_OPERATOR, maxPrice)); + } + List sortParams = new ArrayList(); + sortParams.add(new SortParam("tradeDate", false)); + List dealPrices = dealPriceDao.findAllByParams(params, sortParams, limit); + return dealPrices; + } + @Override public List getDisSameDealPrice(DistrictKey key, int minPrice, int maxPrice, int limit) { List params = new ArrayList(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java new file mode 100644 index 000000000..bc8c159de --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityBasePriceService.java @@ -0,0 +1,9 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityBasePriceService { + + CommunityBasePrice getById(String id); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java new file mode 100644 index 000000000..c0eb458f1 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/CommunityPoiService.java @@ -0,0 +1,8 @@ +package com.fangjia.service.open; + +import com.fangjia.dal.mongo.open.model.CommunityPoi; + +public interface CommunityPoiService { + + CommunityPoi getById(String id); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java new file mode 100644 index 000000000..049e5ae16 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityBasePriceServiceImpl.java @@ -0,0 +1,22 @@ +package com.fangjia.service.open.impl; + +import com.fangjia.dal.mongo.open.dao.CommunityBasePriceDao; +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityBasePrice; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.service.open.CommunityBasePriceService; +import com.fangjia.service.open.CommunityPoiService; + +public class CommunityBasePriceServiceImpl implements CommunityBasePriceService { + + private CommunityBasePriceDao communityBasePriceDao; + + @Override + public CommunityBasePrice getById(String id) { + return communityBasePriceDao.findById(id); + } + + public void setCommunityBasePriceDao(CommunityBasePriceDao communityBasePriceDao) { + this.communityBasePriceDao = communityBasePriceDao; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java new file mode 100644 index 000000000..8ea0b05e1 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/impl/CommunityPoiServiceImpl.java @@ -0,0 +1,19 @@ +package com.fangjia.service.open.impl; + +import com.fangjia.dal.mongo.open.dao.CommunityPoiDao; +import com.fangjia.dal.mongo.open.model.CommunityPoi; +import com.fangjia.service.open.CommunityPoiService; + +public class CommunityPoiServiceImpl implements CommunityPoiService { + + private CommunityPoiDao communityPoiDao; + + @Override + public CommunityPoi getById(String id) { + return communityPoiDao.findById(id); + } + + public void setCommunityPoiDao(CommunityPoiDao communityPoiDao) { + this.communityPoiDao = communityPoiDao; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/util/CookieUtils.java b/projects/fangjia-service/src/main/java/com/fangjia/service/util/CookieUtils.java index 79fe7a4ed..1a3f74114 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/util/CookieUtils.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/util/CookieUtils.java @@ -1,5 +1,7 @@ package com.fangjia.service.util; +import org.apache.commons.lang.StringUtils; + import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -29,6 +31,29 @@ public class CookieUtils { } + public static boolean removeCookie(HttpServletRequest request, HttpServletResponse response, String name) { + if(request.getCookies() == null){ + return false; + } + if (StringUtils.isBlank(name)) { + return false; + } + Cookie[] cookies = request.getCookies(); + if (null != cookies) { + for (Cookie cookie : cookies) { + if (name.equals(cookie.getName())) { + cookie.setValue(null); + cookie.setMaxAge(0); + cookie.setPath("/"); + response.addCookie(cookie); + return true; + } + } + } + return false; + } + + public static void setCookie(String name, String value, String path,int maxAge, HttpServletResponse response){ Cookie cookie = new Cookie(name, value); if(path != null){ diff --git a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css index 0b498e441..805853bc9 100644 --- a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css +++ b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/company.css @@ -20,7 +20,7 @@ .officeMap{width:100%;height:400px;} .main_content.partner{overflow:hidden;} -.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:24px;margin-bottom:25px;cursor:pointer;} +.main_content.partner a{float:left;width:180px;height:120px;background:url(../../../images/fangjia/v5/partner.png) no-repeat #f0f4f7;margin-right:15px;margin-bottom:25px;cursor:pointer;} .partner a.partner11{background-position:0 0;} .partner a.partner12{background-position:-204px 0;} diff --git a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/evaluation.css b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/evaluation.css index 76562e090..7ad6783ae 100644 --- a/projects/fangjia-static/src/main/webapp/style/fangjia/v5/evaluation.css +++ b/projects/fangjia-static/src/main/webapp/style/fangjia/v5/evaluation.css @@ -424,7 +424,7 @@ .trend01a p small{display:block;text-align:center;font-size:12px;color:#999;margin-bottom:10px;} .trend01b{overflow:hidden;} -.trend01b div{width:350px;height:110px;background:url(../../../images/fangjia/v5/evaluation_line.png);border:solid 1px #e7ecf1;padding-top:30px;float:left;margin-right:72px;} +.trend01b div{width:20%;height:110px;background:url(../../../images/fangjia/v5/evaluation_line.png);border:solid 1px #e7ecf1;padding-top:30px;float:left;margin-right:72px;} .trend01b div label{display:block;text-align:center;color:#666;line-height:40px;} .trend01b div em{display:block;text-align:center;font-size:36px;}