diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/ApiUserController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/ApiUserController.java index 7c782d610..7df83ddd3 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/ApiUserController.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/ApiUserController.java @@ -102,6 +102,7 @@ public class ApiUserController { bo.setPurpose(apiUser.getPurpose()); bo.setDeadTime(apiUser.getDeadTime()); bo.setManager(apiUser.getManager()); + bo.setIsMonitor(apiUser.getIsMonitor()); apiUserBoList.add(bo); } @@ -183,6 +184,7 @@ public class ApiUserController { apiUser.setModificationDate(date); apiUser.setDeadTime(model.getDeadTime()); apiUser.setManager(model.getManager()); + apiUser.setIsMonitor(model.getIsMonitor()); this.apiUserService.save(apiUser); if(model.getEvaluateRate() != null && model.getEvaluateRate() != 0){ ApiUserCfg apiUserCfg = new ApiUserCfg(); @@ -222,6 +224,7 @@ public class ApiUserController { bo.setEmail(apiUser.getEmail()); bo.setDeadTime(apiUser.getDeadTime()); bo.setManager(apiUser.getManager()); + bo.setIsMonitor(apiUser.getIsMonitor()); ApiUserCfg apiUserCfg_rate = apiUserCfgService.findByUserId(apiUser.getId(), ApiUserCfg.ApiUserCfgKey.EVALUATE_RATE); if(apiUserCfg_rate != null){ bo.setEvaluateRate((Double) apiUserCfg_rate.getCfgValue()); @@ -272,6 +275,7 @@ public class ApiUserController { dataApiUser.put("purpose", model.getPurpose()); dataApiUser.put("deadTime", model.getDeadTime()); dataApiUser.put("manager", model.getManager()); + dataApiUser.put("isMonitor", model.getIsMonitor()); this.apiUserService.update(dataApiUser, userId); //ApiUserCfg.ApiUserCfgKey.EVALUATE_RATE diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java new file mode 100644 index 000000000..afd4fb55c --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/BuildingRatioController.java @@ -0,0 +1,167 @@ +package com.fangjia.bkoff.controllers; + +import com.fangjia.bkoff.model.ApiUserModel; +import com.fangjia.bkoff.model.BuildingRatioModel; +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; +import com.fangjia.dal.sql.model.User; +import com.fangjia.service.bo.BuildingRatioBo; +import com.fangjia.service.mongo.bkoff.BuildingRatioService; +import org.apache.commons.lang.StringUtils; + +import java.net.URLEncoder; +import java.util.*; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioController { + + private BuildingRatioService buildingRatioService; + + /** + * 新建楼栋系数信息 + * + * @param model + * @return + */ + public Object doAddAction(BuildingRatioModel model) throws Exception { + try { + if(checkBuildingNameAction(model)){ + model.redirect("/buildingRatio/add?msg=" + URLEncoder.encode(model.getMsg(), "UTF-8")); + return model; + } + Date date = new Date(); + BuildingRatio buildingRatio = new BuildingRatio(); + buildingRatio.setCity(model.getCity_name()); + buildingRatio.setRegion(model.getRegion()); + buildingRatio.setDistrict(model.getDistrict()); + buildingRatio.setBuilding_name(model.getBuilding_name()); + buildingRatio.setRatio(model.getRatio()); + buildingRatio.setC_date(date); + buildingRatio.setM_date(date); + buildingRatio.setCreate_user(model.getCreate_user()); + buildingRatio.setNote(model.getNote()); + this.buildingRatioService.save(buildingRatio); + + model.setMsg("新建成功!"); + } catch (Exception e) { + e.printStackTrace(); + model.setMsg("新建失败!"); + } + model.redirect("/buildingRatio/add?msg=" + URLEncoder.encode(model.getMsg(), "UTF-8")); + return model; + } + + public Object doDeleteAction(BuildingRatioModel model){ + if(StringUtils.isNotBlank(model.getId())){ + this.buildingRatioService.deleteById(model.getId()); + } + model.setViewType("redirect"); + model.setViewParam("location","/buildingRatio/list"); + return model; + } + + /** + * 修改楼栋系数信息 + * + * @param model + * @return + */ + public Object updateAction(BuildingRatioModel model) throws Exception { + Map data_buildingRatio = new HashMap(); + + data_buildingRatio.put("id", model.getId()); + data_buildingRatio.put("city", model.getCity_name()); + data_buildingRatio.put("region", model.getRegion()); + data_buildingRatio.put("district", model.getDistrict()); + data_buildingRatio.put("building_name", model.getBuilding_name()); + data_buildingRatio.put("ratio", model.getRatio()); + data_buildingRatio.put("m_date", new Date()); + data_buildingRatio.put("create_user", model.getCreate_user()); + data_buildingRatio.put("note", model.getNote()); + + this.buildingRatioService.saveOrUpdateByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name(), data_buildingRatio); + + model.setViewType("redirect"); + model.setViewParam("location", "/buildingRatio/list"); + + return model; + } + + public Object listAction(BuildingRatioModel model) { + + List buildingRatioList = this.buildingRatioService.findAllByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name()); + + List buildingRatioBoList = new ArrayList(); + + for (BuildingRatio buildingRatio : buildingRatioList) { + BuildingRatioBo bo = new BuildingRatioBo(); + + bo.setId(buildingRatio.getId()); + bo.setCity_name(buildingRatio.getCity()); + bo.setRegion(buildingRatio.getRegion()); + bo.setDistrict(buildingRatio.getDistrict()); + bo.setBuilding_name(buildingRatio.getBuilding_name()); + bo.setRatio(buildingRatio.getRatio()); + bo.setC_date(buildingRatio.getC_date()); + bo.setM_date(buildingRatio.getM_date()); + bo.setCreate_user(buildingRatio.getCreate_user()); + bo.setNote(buildingRatio.getNote()); + + buildingRatioBoList.add(bo); + } + + model.setBuildingRatioBoList(buildingRatioBoList); + return model; + } + + public Object addAction(BuildingRatioModel model) { + return model; + } + + /** + * 编辑api用户信息 + * + * @param model + * @return + */ + public Object editAction(BuildingRatioModel model) { + BuildingRatio buildingRatio = this.buildingRatioService.findBuildingRatioById(model.getId()); + + BuildingRatioBo bo = new BuildingRatioBo(); + bo.setId(buildingRatio.getId()); + bo.setCity_name(buildingRatio.getCity()); + bo.setRegion(buildingRatio.getRegion()); + bo.setDistrict(buildingRatio.getDistrict()); + bo.setBuilding_name(buildingRatio.getBuilding_name()); + bo.setRatio(buildingRatio.getRatio()); + bo.setCreate_user(buildingRatio.getCreate_user()); + bo.setNote(buildingRatio.getNote()); + + model.setBuildingRatioBo(bo); + if (StringUtils.isNotBlank(model.getMsg()) && model.getMsg().equals("success")) { + model.setMsg("更新成功!"); + } + return model; + } + + public boolean checkBuildingNameAction(BuildingRatioModel model) { + model.setViewType("json"); + model.setViewParam("config", null); + List buildingRatioList = buildingRatioService.findAllByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name()); + if (buildingRatioList.size()>0) { + model.setMsg("此楼栋名称已经存在!"); + return true; + } + return false; + } + + public BuildingRatioService getBuildingRatioService() { + return buildingRatioService; + } + + public void setBuildingRatioService(BuildingRatioService buildingRatioService) { + this.buildingRatioService = buildingRatioService; + } +} diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/ApiUserModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/ApiUserModel.java index b252774f1..ad264fe9a 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/ApiUserModel.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/ApiUserModel.java @@ -38,6 +38,7 @@ public class ApiUserModel extends BaseModel { private Integer creditLevelStatus; private String priceType; private Map priceRatingMap; + private Integer isMonitor; public String getPriceType() { return priceType; @@ -606,4 +607,12 @@ public class ApiUserModel extends BaseModel { public void setManager(String manager) { this.manager = manager; } + + public Integer getIsMonitor() { + return isMonitor; + } + + public void setIsMonitor(Integer isMonitor) { + this.isMonitor = isMonitor; + } } diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java new file mode 100644 index 000000000..7afdeb9db --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/model/BuildingRatioModel.java @@ -0,0 +1,134 @@ +package com.fangjia.bkoff.model; + +import com.fangjia.service.bo.BuildingRatioBo; +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; +import java.util.List; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioModel extends BaseModel{ + + private String id; + private String city_name; + private String region; + private String district; + private String building_name; + private String create_user; + private String note; + private Double ratio; + private String msg; + private List buildingRatioBoList; + private BuildingRatioBo buildingRatioBo; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date c_date; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date m_date; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCity_name() { + return city_name; + } + + public void setCity_name(String city_name) { + this.city_name = city_name; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district_name) { + this.district = district_name; + } + + public String getBuilding_name() { + return building_name; + } + + public void setBuilding_name(String building_name) { + this.building_name = building_name; + } + + public Double getRatio() { + return ratio; + } + + public void setRatio(Double ratio) { + this.ratio = ratio; + } + + public Date getC_date() { + return c_date; + } + + public void setC_date(Date c_date) { + this.c_date = c_date; + } + + public Date getM_date() { + return m_date; + } + + public void setM_date(Date m_date) { + this.m_date = m_date; + } + + public List getBuildingRatioBoList() { + return buildingRatioBoList; + } + + public void setBuildingRatioBoList(List buildingRatioBoList) { + this.buildingRatioBoList = buildingRatioBoList; + } + + public BuildingRatioBo getBuildingRatioBo() { + return buildingRatioBo; + } + + public void setBuildingRatioBo(BuildingRatioBo buildingRatioBo) { + this.buildingRatioBo = buildingRatioBo; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } + + public String getCreate_user() { + return create_user; + } + + public void setCreate_user(String create_user) { + this.create_user = create_user; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } +} diff --git a/projects/fangjia-bkoff/src/main/webapp/apiUser/add.ftl b/projects/fangjia-bkoff/src/main/webapp/apiUser/add.ftl index 410b73fae..29ae67e83 100644 --- a/projects/fangjia-bkoff/src/main/webapp/apiUser/add.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/apiUser/add.ftl @@ -57,13 +57,20 @@ 璐︽埛鐢ㄩ�� + + 鏄惁鐩戞帶 + + 涓嶇洃鎺� + 鐩戞帶 + + Email diff --git a/projects/fangjia-bkoff/src/main/webapp/apiUser/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/apiUser/edit.ftl index caf99a2c7..31fb2f51a 100644 --- a/projects/fangjia-bkoff/src/main/webapp/apiUser/edit.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/apiUser/edit.ftl @@ -72,6 +72,13 @@ + + 鏄惁鐩戞帶 + + checked="checked" name="isMonitor" value="0" />涓嶇洃鎺� + checked="checked" name="isMonitor" value="1" style="margin-left: 15px;"/>鐩戞帶 + + Email diff --git a/projects/fangjia-bkoff/src/main/webapp/apiUser/list.ftl b/projects/fangjia-bkoff/src/main/webapp/apiUser/list.ftl index 0924ceb7e..37afef638 100644 --- a/projects/fangjia-bkoff/src/main/webapp/apiUser/list.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/apiUser/list.ftl @@ -53,6 +53,9 @@ 璐︽埛鐢ㄩ�� + + 鏄惁鐩戞帶 + 鍒版湡鏃堕棿 @@ -114,6 +117,13 @@ 鎴跨敓娲� + + <#if bo.isMonitor==1> + 鏄� + <#else> + 鍚� + + <#if bo.deadTime??> ${bo.deadTime?string("yyyy骞碝M鏈坉d鏃�")} diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl new file mode 100644 index 000000000..ca7565298 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/add.ftl @@ -0,0 +1,112 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 妤兼爧绯绘暟鏂板 + + <#include "/common_js.ftl" /> + + + + <#include "/layout/header.ftl" /> +
+${msg!''} + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ 鏍忎綅 + + 鍊� +
鍩庡競
鍖哄煙
灏忓尯鍚嶇О
妤兼爧鍚嶇О
妤兼爧绯绘暟
鍒涘缓浜�
澶囨敞
+
+
+ + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl new file mode 100644 index 000000000..49f7c0cd4 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/edit.ftl @@ -0,0 +1,113 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 妤兼爧绯绘暟淇℃伅淇敼 + + + <#include "/common_js.ftl" /> + + + + + + <#include "/layout/header.ftl" /> +
+ ${msg!''} + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ 鏍忎綅 + + 鍊� +
鍩庡競
鍖哄煙
灏忓尯鍚嶇О
妤兼爧鍚嶇О
妤兼爧绯绘暟
鍒涘缓浜�
澶囨敞
+
+
+ + + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl new file mode 100644 index 000000000..9945afa46 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/webapp/buildingRatio/list.ftl @@ -0,0 +1,120 @@ +<#ftl attributes={"content_type":"text/html; charset=UTF-8"}> +<#compress> + + + + + 鍧囦环绠$悊-妤兼爧绯绘暟绠$悊 + + <#include "/common_js.ftl" /> + + + <#include "/layout/header.ftl" /> + +
+
+ 鍩庡競锛� + 鍖哄煙锛� + 灏忓尯锛� + +
+
+ +
+ ${msg!''} + +
+
+
+ 鍏辨湁妤兼爧绯绘暟${buildingRatioBoList?size}涓� +
+ + + + + + + + + + + + + + + + + <#if (buildingRatioBoList??&&buildingRatioBoList?size>0)> + <#list buildingRatioBoList as bo> + + + + + + + + + + + + + + + +
+ ID + + 鍩庡競 + + 鍖哄煙 + + 灏忓尯鍚嶇О + + 妤兼爧鍚嶇О + + 妤兼爧绯绘暟 + + 鍒涘缓浜� + + 鍒涘缓鏃ユ湡 + + 淇敼鏃ユ湡 + + 澶囨敞 + + 鎿嶄綔 +
+ ${bo.id!} + + ${bo.city_name!} + + ${bo.region!} + + ${bo.district!} + + ${bo.building_name!} + + ${bo.ratio!} + + ${bo.create_user!} + + <#if bo.c_date??> + ${bo.c_date?string("yyyy骞碝M鏈坉d鏃� HH:mm:ss")} + + + <#if bo.m_date??> + ${bo.m_date?string("yyyy骞碝M鏈坉d鏃� HH:mm:ss")} + + + ${bo.note!} + + 缂栬緫 + 鍒犻櫎 +
+
+
+ + + \ No newline at end of file diff --git a/projects/fangjia-bkoff/src/main/webapp/layout/menu.ftl b/projects/fangjia-bkoff/src/main/webapp/layout/menu.ftl index 719e92e83..d44922973 100644 --- a/projects/fangjia-bkoff/src/main/webapp/layout/menu.ftl +++ b/projects/fangjia-bkoff/src/main/webapp/layout/menu.ftl @@ -90,6 +90,7 @@
    <@menuitem uri='/avgPriceUpdate/doCreatTask' text="浣忓畢鍧囦环绠$悊" /> <@menuitem uri='/averagerConfig/list' text="浣忓畢绯绘暟绠$悊" /> + <@menuitem uri='/buildingRatio/list' text="妤兼爧绯绘暟绠$悊" /> <@menuitem uri='/avgPriceUpdate/listQR' text="浠锋牸璐ㄩ噺璺熻釜" /> <@menuitem uri='/avgPriceUpdate/listSQR' text="鏍锋湰璐ㄩ噺璺熻釜" /> <@menuitem uri='/comparePrice/listQR' text="璋冩煡鏁版嵁璺熻釜" /> diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDao.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDao.java new file mode 100644 index 000000000..f530cfff3 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDao.java @@ -0,0 +1,12 @@ +package com.fangjia.dal.mongo.dao.bkoff; + +import com.fangjia.dal.MongoDao; +import com.fangjia.dal.mongo.model.bkoff.AveragerConfig; +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public interface BuildingRatioDao extends MongoDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDaoImpl.java new file mode 100644 index 000000000..192ad0456 --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/dao/bkoff/BuildingRatioDaoImpl.java @@ -0,0 +1,12 @@ +package com.fangjia.dal.mongo.dao.bkoff; + +import com.fangjia.dal.MongoDaoImpl; +import com.fangjia.dal.mongo.model.bkoff.AveragerConfig; +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioDaoImpl extends MongoDaoImpl implements BuildingRatioDao { +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java new file mode 100644 index 000000000..8868f4c4f --- /dev/null +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/mongo/model/bkoff/BuildingRatio.java @@ -0,0 +1,113 @@ +package com.fangjia.dal.mongo.model.bkoff; + +import com.fangjia.dal.MongoEntity; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; +import javax.persistence.Table; + +import org.springframework.format.annotation.DateTimeFormat; + +import java.util.Date; + + + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +@Entity +@Table(name = "business.building_ratio") +public class BuildingRatio implements MongoEntity { + + private String id; + private String city; + private String region; + private String district; + private String building_name; + + private Double ratio; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date c_date; + @DateTimeFormat(pattern = "yyyy-MM-dd") + private Date m_date; + private String create_user; + private String note; + + @Id + @GeneratedValue + @Column(name = "_id") + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + @Column(name = "city") + public String getCity() { + return city; + } + public void setCity(String city) { + this.city = city; + } + @Column(name = "region") + public String getRegion() { + return region; + } + public void setRegion(String region) { + this.region = region; + } + @Column(name = "district") + public String getDistrict() { + return district; + } + public void setDistrict(String district) { + this.district = district; + } + @Column(name = "building_name") + public String getBuilding_name() { + return building_name; + } + public void setBuilding_name(String building_name) { + this.building_name = building_name; + } + @Column(name = "ratio") + public Double getRatio() { + return ratio; + } + public void setRatio(Double ratio) { + this.ratio = ratio; + } + @Column(name = "c_date") + public Date getC_date() { + return c_date; + } + public void setC_date(Date c_date) { + this.c_date = c_date; + } + @Column(name = "m_date") + public Date getM_date() { + return m_date; + } + public void setM_date(Date m_date) { + this.m_date = m_date; + } + @Column(name = "create_user") + public String getCreate_user() { + return create_user; + } + + public void setCreate_user(String create_user) { + this.create_user = create_user; + } + @Column(name = "note") + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } +} diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ApiUser.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ApiUser.java index 1cddeac31..52806e4df 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ApiUser.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/ApiUser.java @@ -39,6 +39,8 @@ public class ApiUser implements SqlEntity { private String manager; + private Integer isMonitor; + @Id @Column(name = "ID") public long getId() { @@ -175,4 +177,12 @@ public class ApiUser implements SqlEntity { public void setManager(String manager) { this.manager = manager; } + @Column(name = "IS_MONITOR") + public Integer getIsMonitor() { + return isMonitor; + } + + public void setIsMonitor(Integer isMonitor) { + this.isMonitor = isMonitor; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index f34d40421..1c1fa2788 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -51,7 +51,19 @@ for(def d : datas){ map.region = d['B'] def roomStr = d['E'] if(roomStr){ - map.room = roomStr.toString().toDouble().toInteger() + if(roomStr.contains("-")){ + def arrs = roomStr.split("-") + try{ + map.room = arrs[0].toString().toInteger() + }catch(Exception e){ + println "room error" + } + map.hall = arrs[1].toString().toInteger() + map.toilet = arrs[3].toString().toInteger() + }else{ + map.room = roomStr.toString().toDouble().toInteger() + } + } map.area = d['F'].toString().toDouble() @@ -59,7 +71,13 @@ for(def d : datas){ map.avg_price = (int)(d['H'].toString().toDouble()) map.sell_price = (int)(d['I'].toString().toDouble()*10000) def trade_date = d['J'] - map.trade_date = trade_date + if(trade_date){ + map.trade_date = trade_date + }else{ + def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31") + map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") + } + map.source = "深房家族" map.create_date = new Date() map.m_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 17860e977..61b833dab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -77,9 +77,8 @@ class BigdataCommunityBatch extends BaseBatch{ FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成->导Oracle" - -// delPgData() -// saveToPgSql(rows) + delPgData() + saveToPgSql(rows) // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 4d289d469..13fcabfc3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -33,7 +33,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")); + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM")); def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); @@ -61,17 +61,19 @@ class BigdataDealBatch extends BaseBatch{ public void process(Object args) { //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; - def citys = CcbCommonUtils.getRunningCitys(args) +// def citys = CcbCommonUtils.getRunningCitys(args) if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } println "curDate:"+curDate - def allCitys = CommonDatautils.findAllCity() + def allCitys = ["福州"] //CommonDatautils.findAllCity() for(def city : allCitys){ // if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); + addDealPriceMatch(city); // } } + + // def rows = [] // def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] // def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -649,16 +651,21 @@ class BigdataDealBatch extends BaseBatch{ def final AtomicInteger count_ = new AtomicInteger(0) for(def i=0;i + def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306] + def deal_pes = [] + def third_pes = [] + for(def d : dates){ + def map = findPeGteCity(row['B'], d) + if(map.deal_pe){ + deal_pes << map.deal_pe + } + if(map.third_pe){ + third_pes << map.third_pe + } + } + if(deal_pes){ + row['F'] = deal_pes.sum()/deal_pes.size() + } + if(third_pes){ + row['G'] = third_pes.sum()/third_pes.size() + } + println row['B'] + " "+row['F']+" "+row['G'] + }) + + } + + + + def findPeGteCity(cityName, yearmonth){ + def sql = "SELECT gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}" + def map = [:] + pgsqlInstance.eachRow(sql,{r-> + map.deal_pe = r.gujiamdltst_ppe15_pct + map.third_pe = r.gujiatrdpttstppe15pct + }); + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 91fb02cdc..80f739637 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-27") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy index f2a564cb7..2fd7e0c2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/PgsqlCommUtils.groovy @@ -17,4 +17,14 @@ class PgsqlCommUtils { }); return set } + + + public static def findAllDistrictCount(def cityid){ + def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0"; + def count = 0 + sqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index db8fef0a1..0fd8eaa49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -33,8 +33,8 @@ import java.util.regex.*; def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") -def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") @@ -45,8 +45,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) -//def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", -// "postgres", "postgres", "org.postgresql.Driver") +def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") //def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"] //def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() @@ -1287,50 +1287,50 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx" -// +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx" + //ExcelUtils.multiThreadingModify(fileName, {d-> // if(d['A']){ -//// def region = RegionData.formatRegion(d['A'], d['B']) -//// if(region){ -//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district") -//// if(dist){ -//// d['E'] = dist.region -//// d['F'] = dist._id.toString() -//// } -//// } -// if(!d['F']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -//// +// def region = RegionData.formatRegion(d['B'], d['C']) +// if(region){ +// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") // if(dist){ -//// d['H'] = dist.region -//// d['G'] = dist.name -// d['F'] = dist.status +// d['G'] = dist.region +// d['H'] = dist._id.toString() // } // } +//// if(!d['F']){ +//// def dist = CommonDatautils.findDistrictById(d['A']) +////// +//// if(dist){ +////// d['H'] = dist.region +////// d['G'] = dist.name +//// d['F'] = dist.status +//// } +//// } // // } //},0) -def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx" -ExcelUtils.multiThreadingModify(fileName, { d -> - def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district") - if(count && count==1){ - def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district") - d['E'] = dist.city - } - -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// d['B'] = dist.region -// d['C'] = dist.name -// d['J'] = dist.status +//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx" +//ExcelUtils.multiThreadingModify(fileName, { d -> +// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district") +// if(count && count==1){ +// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district") +// d['E'] = dist.city // } -// def newRegion = RegionData.formatRegion(d['A'], d['B']) -// d['F'] = newRegion -},0) +// +//// def dist = CommonDatautils.findDistrictById(d['A']) +//// if(dist){ +//// d['B'] = dist.region +//// d['C'] = dist.name +//// d['J'] = dist.status +//// } +//// def newRegion = RegionData.formatRegion(d['A'], d['B']) +//// d['F'] = newRegion +//},0) //ExcelUtils.modify(fileName, {d-> // if(d['A']){ @@ -1364,6 +1364,53 @@ def findGuidedPrice(sqlInstance, id){ return count } +def findPgDealPrice(c, pgsqlInstance, startDate){ + def sql = "select count(a.gujia_comm_id) from (SELECT distinct gujia_comm_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${c}' and txn_dt>='${startDate}') a" + def count = 0 + pgsqlInstance.eachRow(sql,{r-> + count = r.count + }); + return count +} + +citys = CcbCommonUtils.citys_all +def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"] +def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306] +def rows = [] +def headers = [] +headers << "城市" +headers.addAll(dates) +rows << headers +for(def c : citys){ + def row = [] + row << c + def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c)) +// for(def d : dates){ + def startDate = "2022-07-01" + def count = findPgDealPrice(c, pgsqlInstance, startDate) + row << count*100/disCount + +// def map = findSampleRate(pgsqlInstance, c, d) +// row << map.third +// } + rows << row +} +ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方") + + + + + +def findSampleRate(pgsqlInstance, cityName, yearmonth){ + def sql = "SELECT urbn_nm,gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}" + def map = [:] + pgsqlInstance.eachRow(sql,{r-> + map.deal = r.gujiamdltst_ppe15_pct + map.third = r.gujiatrdpttstppe15pct + }); + return map +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index da8662b0e..492fb152d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); saveToPgSql(rows) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index e8bd49824..9156b9d6d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -45,6 +45,7 @@ class TradeStatLog { // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() + // task.deleteDistrict() // task.deleteDealData() // task.deleteDealData2() @@ -92,13 +93,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-02-01" - def date = "2023-03-01" - def endDate = "2023-04-01" - def newDate = "2023-05-01" + def startDate = "2023-03-01" + def date = "2023-04-01" + def endDate = "2023-05-01" + def newDate = "2023-06-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy index d7be2d646..3b095eb3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDataStat.groovy @@ -19,9 +19,9 @@ class CcbDataStat { static main(args) { def task = new CcbDataStat() -// task.run() + task.run(args) // task.tradeRun() - task.regionDistStat() +// task.regionDistStat() } def regionDistStat(){ @@ -124,22 +124,37 @@ class CcbDataStat { return count } - def run(){ + def run(args){ + def citys = CcbCommonUtils.getRunningCitys(args) def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d-> - if(d['A']) { - def citycode = cityCodeDataMap.get(d['A']) - def total = findTotalCount(citycode) - def f_count = findDistFinishDate(citycode) - def b_count = findBuildTypeCount(citycode) - def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total) - def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total) - d['C'] = f_pre - d['D'] = b_pre - d['E'] = total - println d['A'] +" "+ total +" "+f_count+" "+b_count - } - }) + def rets = [] + rets << ["城市","经纬度","物业类型","建筑年代","建筑类型","绿化率","容积率"] + for(def c : citys){ + def ret = [] + ret << c + def citycode = cityCodeDataMap.get(c) + def total = findTotalCount(citycode) + def lat_count = findLocationCount(citycode) + def f_count = findDistFinishDate(citycode) + def b_count = findBuildTypeCount(citycode) + def g_count = findVirescencePercent(citycode) + def c_count = findCubagePercentPercent(citycode) + def location_pre = DataHandleUtils.getDoubleRound(2, lat_count*100/total) + def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total) + def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total) + def g_pre = DataHandleUtils.getDoubleRound(2, g_count*100/total) + def c_pre = DataHandleUtils.getDoubleRound(2, c_count*100/total) + ret << location_pre + ret << 100 + ret << b_pre + ret << f_pre + ret << g_pre + ret << c_pre + ret << total + rets << ret + } + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\建行供数重要字段覆盖率.xlsx", rets, "统计") } def findTotalCount(citycode){ @@ -168,4 +183,32 @@ class CcbDataStat { }) return count } + + def findLocationCount(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and longitude>0";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } + + def findVirescencePercent(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and GREENINGRATE is not null";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } + + + def findCubagePercentPercent(citycode){ + def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and PLOTRATIO is not null";//where cityId in (350200,150100) + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + return count + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy index ed89f59ac..9ed8300cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCitypriceBatch.groovy @@ -61,9 +61,9 @@ class BigdataCitypriceBatch extends BaseBatch{ def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1) def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger(); - cityRun(citys, startDate, endDate, cityCodeMap) - regionRun(citys, startDate, endDate, cityCodeMap) -// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) +// cityRun(citys, startDate, endDate, cityCodeMap) +// regionRun(citys, startDate, endDate, cityCodeMap) + districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap) } def cityRun(citys, startDate, endDate, cityCodeDataMap){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index ae9f4f058..c8d486778 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -1184,15 +1184,15 @@ class ExportDistrictRongHe extends BaseBatch{ } def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = [] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); - println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" CcbFtpUtils.ftpUpload("rh", fileName, curDate) - println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" } @@ -1273,9 +1273,9 @@ class ExportDistrictRongHe extends BaseBatch{ def exportPgCommunityPdescribeGj(month){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" - def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat" } def headers = [] def allRows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy index 0d0e263e6..25560ff03 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/chongqing/HailanTest.groovy @@ -40,19 +40,26 @@ class HailanTest { static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + static main(args) { def task = new HailanTest(); task.run(); } def run(){ - def inputfile = "D:\\数据\\微元素\\排查表.xlsx" - match(inputfile) + def inputfile = "D:\\数据\\微元素\\排查表-价格对比-新.xlsx" +// addThirdPrice(inputfile) +// match(inputfile) +// tradeAvgPrice(inputfile) +// caluateTotalPrice(inputfile) // addPrice(inputfile) // addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx") // addHouse(inputfile) // stat(inputfile) // hailanStat(inputfile) +// addFzFjwPrice(inputfile) + addFjwTobPrice(inputfile) } def addHouse(inputfile){ @@ -702,4 +709,72 @@ class HailanTest { } return prices; } + + def tradeAvgPrice(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){ + def dist = CommonDatautils.findDistrictById(row['I']) + if(dist){ + def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01"); + row['J'] = (int)tradeAvgPrice + } + } + }) + } + + + def addThirdPrice(path){ + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 + + + ExcelUtils.multiThreadingModify(path, { row -> + def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate) + def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G']) + if(priceMap){ + if(priceMap.containsKey("lianjia_esf_price")){ + row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95) + } + } + }) + } + + def caluateTotalPrice(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(row['J']){ + row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble()) + }else if(row['K']){ + row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble()) + } + }) + } + + + def addFjwTobPrice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(d['F']){ + def dbo = CommonDatautils.queryDistrictPrice("福州", d['F'], d['G'], DateUtils.parseFromTightByPattern("2023-07-02","yyyy-MM-dd")) + if(dbo){ + d['Q'] = dbo.avg_price_2b + } + } + }) + } + + def addFzFjwPrice(inputfile){ + ExcelUtils.modify(inputfile, { row -> + if(row['B']) { + def q = new BasicDBObject() + q.put("city", "福州") + q.put("region", row['A']) + q.put("name", row['B']) + q.put("month", 202304) + def obj = FZ_DISTRICT_STAT_COLL.findOne(q) + if (obj) { + row['D'] = obj.avg_price + } + } + + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 2ec846add..4540e31c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.commData +import com.fangjia.core.RegionData import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import groovy.json.JsonSlurper import net.sf.json.JSONArray @@ -250,7 +252,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "长滩美墅三期") +// q.put("name", "鲁能大厦") DBObject keys = new BasicDBObject(); if(fields){ @@ -737,8 +739,9 @@ class CommonDatautils { public static def findAllDistrict(def cat, def fields){ DBObject q = new BasicDBObject(); - q.put("visible",0) + q.put("status",0) q.put("cat", cat); +// q.put("name", "中兴智汇大厦") DBObject keys = new BasicDBObject(); if(fields){ @@ -746,7 +749,8 @@ class CommonDatautils { keys.put(field, 1); } } - def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) + def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000) +// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) def rets = [] for(def d : dbsour){ rets << d @@ -1022,7 +1026,7 @@ class CommonDatautils { q.put("cat", cat); q.put("region", region) q.put("name", name); - q.put("status", 0) + q.put("visible", 0) DBObject keys = new BasicDBObject(); if(fields){ @@ -3801,5 +3805,116 @@ class CommonDatautils { } + static def findTrades(dist, startDate, endDate){ + def q = new BasicDBObject() + q.put("city", dist.city) + q.put("region", dist.region) + + if(dist.alias){ + if(!dist.alias.contains(dist.name)){ + dist.alias << dist.name + } + q.put("district_name", new BasicDBObject('$in', dist.alias)) + }else{ + q.put("district_name", dist.name) + } + + q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate))) + def dbcorsor = DEAL_PRICE_COLL.find(q) + def avg_price = 0 + def total_price = 0 + def count = 0 + + for(def d : dbcorsor){ + if(d.avg_price){ + total_price = total_price + d.avg_price + count++ + } + } + if(count > 3){ + avg_price = total_price/count + } + return avg_price + } + + static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor){ + def key = db.city+"##"+db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.lianjia_esf_price) + } + if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price) + } + + if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("lianjia_esf_price", db.beike_esf_price) + } + } + return map + } + + + /** + * 根据经纬度获取区域 + * @param city + * @param location + * @return + */ + static def getRegionByLocation(city, def location){ + def json = GaoDeApiUtils.geocode(location); + def region = null; + if (json && json.info=="OK") { + def regeocode = json.regeocode + if(regeocode && regeocode.addressComponent){ + region = regeocode.addressComponent.district + if(!region){ + region = regeocode.addressComponent.township + } + } + } + if(region){ + region = RegionData.formatRegion(city, region) + } + return region; + } + + def static getAllFiles(def path) { + + File file = new File(path); + File[] tempList = file.listFiles(); + + def files = []; + for (int i = 0; i < tempList.length; i++) { + if (tempList[i].isFile()) { + files << tempList[i]; + } + if (tempList[i].isDirectory()) { + System.out.println("文件夹:" + tempList[i]); + } + } + return files; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy index 7fe3204dd..5edea094e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/common/DataHandleUtils.groovy @@ -250,8 +250,23 @@ class DataHandleUtils { return sum; } - - - + + + public static isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街道|大道|巷|胡同|街|坊]\\d+(-\\d+)?(号|弄)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } + + + + + static main(args) { + def a = isStandardRoad("杨桥西路236号") + println a + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy new file mode 100644 index 000000000..530d8b36d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/district/MasterNameInAlias.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.district + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.mongodb.BasicDBObject +//删除别名是另一个小区的主名 +class MasterNameInAlias { + + static main(args) { + def task = new MasterNameInAlias() + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity().reverse() + def allList = [] + for(def city : citys){ + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"]) + + def cityList = [] + cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"] + for(def dist1 : disticts){ + def distList = [] + for(def dist2 : disticts){ + if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){ + if(dist2.alias && dist2.alias.contains(dist1.name)){ + def distance = '' + if(dist1.lng && dist2.lng){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d); + } + distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance] + dist2.alias.remove(dist1.name) + def d = new BasicDBObject() + d.put("_id", dist2._id) + + def udata = new BasicDBObject() + udata.put("alias", dist2.alias) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(d, new BasicDBObject('$set', udata)) + } + } + } + if(distList){ + cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', ''] + cityList.addAll(distList) +// cityList << [] + } + } + + if(cityList){ + allList.addAll(cityList) + println city +" "+cityList.size() + } + } +// ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy new file mode 100644 index 000000000..cecc0536d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/AmapAddressUpdate.groovy @@ -0,0 +1,616 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.EsUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.sql.ResultSetMetaData +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class AmapAddressUpdate { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static main(args) { + def task = new AmapAddressUpdate() + def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx" + def addr = task.formatAddress("福州", "北三环路201号") + println addr +// task.run(path) +// task.similarMatch(path) +// task.addressRun(path) +// task.addressSource(path) +// task.amapRun() + +// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较 +// task.updateData(path) +// task.updateLocation(path)//更新经纬度 + } + def run(path){ + def citys = ["福州"]//CommonDatautils.findAllCity() + def allrets = [] + allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"] + for (def city : citys) { + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"]) + def rets = findAmapDetail(disticts) + allrets.addAll(rets) + } + ExcelUtils.write(path, allrets, "数据") + } + + def amapRun(){ + def city = "福州市" + def datas = findAllAmapByCity(city) +// datas = datas[0..10] + println "Es总量:"+datas.size() + def rets = match(datas) + + ExcelUtils.write("d:\\数据\\小区\\福州高德数据匹配.xlsx", rets, "数据") + } + + def match(datas){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + rets << ["ID","高德城市","高德区域","高德名称","高德地址","匹配城市","匹配区域","匹配名称","匹配类型","匹配ID"] + for(def i=0;i +// println data.adname+" "+data.name +" "+data.address +" "+data.type +// if(name == data.name && data.adname.contains(region) && data.type.contains("type")){ +// data.formatAddress = AddressUtil.formatAddress(city, data.address) +// loc = data +// } +// }) + return map; + } + + + + + + def addressRun(inputfile){ + + ExcelUtils.multiThreadingModify(inputfile, { row -> +// if(row['H'] && row['H'].contains("口步行") && row['H'].contains("(")){ +// row['I'] = row['H'].substring(0, row['H'].indexOf("(")) +// } + if(row['J']){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def distance = null + if(dist1){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, row['J'].toString().toDouble(), row['K'].toString().toDouble()); + } + row['M'] = distance + } + + def gaodeRoadFlag = false + if(row['I']){ + gaodeRoadFlag = DataHandleUtils.isStandardRoad(row['I']) + } + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(!row['N'] && row['D'] && row['I']){ + if(row['D']==row['I']){ + row['N'] = 100 + }else if(row['E'] && gaodeRoadFlag && row['E'].contains(row['I'])){ + row['N'] = 1 + } + } + + if(!row['N'] && !dbRoadFlag && gaodeRoadFlag){ + row['N'] = 2 + } + + if(!row['N'] && dbRoadFlag && gaodeRoadFlag && row['I'].contains(row['D'])){ + row['N'] = 3 + } + + if(!row['N'] && dbRoadFlag && gaodeRoadFlag){ + row['N'] = 4 + } + + }) + } + + def addressSource(inputfile) { + + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(!row['N'] && row['D'] && !row['O']){ + def addresses = findAllEsData(row['A'], row['B'], row['C']) + if(addresses){ + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(dbRoadFlag){ + def list = [] + list << row['D'] + if(row['E']){ + def arr = row['E'].split(" ") + list.addAll(arr as List) + } + for(def addr : list){ + if(addresses.contains(addr)){ + row['O'] = "诸葛找房" + break + } + } + } + } + } + }) + } + + + def findAllEsData(city,region, name){ + def rets = [] + def sql = "select * from community where city='${city}' and name='${name}' and source='诸葛找房APP'" + rets = findDatas(region, sql) + if(!rets){ + rets = findAliasDatas(city,region, name) + } + return rets; + } + + def findDatas(region, sql){ + def rets = [] + ElasticUtils.executeSqlQuery(sql, { d-> + if(d.region && d.region.contains(region) && d.address){ + rets << d.address + } + }) + return rets; + } + + + def findAliasDatas(city, region, name){ + def must = [] + def term1 = ["match":["city":city]] + must << term1 + + def should = [] + should << ["match":["alias":name]] + def params = ["query":["bool":["must":must, "should":should]]] + def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/community/_doc/_search"); + def rets = []; + if(s) { + def doc = new groovy.json.JsonSlurper().parseText(s) + def hits = doc.hits + if(hits){ + hits = hits.hits + for(def h : hits){ + def d = h._source + if(d.region && d.region.contains(region) && d.address && d.source=="诸葛找房APP"){ + rets << d.address + } + } + } + } + return rets; + } + + def addrAndLocationDistance(inputfile){ + ExcelUtils.multiThreadingModify(inputfile, { row -> + if(!row['N'] && row['D']){ + println row['D'] + def dbRoadFlag = false + if(row['D']){ + dbRoadFlag = DataHandleUtils.isStandardRoad(row['D']) + } + if(dbRoadFlag){ + def gloJson = GaoDeApiUtils.geo(row['A'], row['D']) + if(gloJson) { + def glocation = null + if (gloJson && gloJson.get("geocodes")) { + def geocodes = gloJson.get("geocodes") + def map = geocodes.get(0) + if(map.formatted_address.contains(row['D'])){ + def arrs = map.location.split(",") + + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def distance = null + if(dist1){ + distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, arrs[0].toString().toDouble(), arrs[1].toString().toDouble()); + } + row['P'] = distance + } + } + } + } + } + }) + } + + def updateData(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(row['Q'] && row['Q'].contains("修改地址")){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + def udata = new BasicDBObject() + if(row['Q'].contains("可替换原地址")){ + udata.put("address", row['I']) + }else if(row['Q'].contains("可新增地址别名")){ + udata.put("address", row['I']) + def addrAlias = dist1.addr_alias + if(!addrAlias){ + addrAlias = [] + } + addrAlias << dist1.address + udata.put("addr_alias", addrAlias) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata)) + } + + + + }) + } + + def updateLocation(path){ + ExcelUtils.multiThreadingModify(path, { row -> + + if(row['R'] && row['R']=="是"){ + def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + if(dist1){ + def udata = new BasicDBObject() + def lat = null + def lng = null + if(row['J']){ + lng = row['J'].toString().toDouble() + lat = row['K'].toString().toDouble() + }else{ + def gloJson = GaoDeApiUtils.geo(row['A'], row['D']) + if(gloJson) { + if (gloJson && gloJson.get("geocodes")) { + def geocodes = gloJson.get("geocodes") + def map = geocodes.get(0) + if (map.formatted_address.contains(row['D'])) { + def arrs = map.location.split(",") + lng = arrs[0].toString().toDouble() + lat = arrs[1].toString().toDouble() + } + } + } + } + + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + def lng2 = baidu.x + def lat2 = baidu.y + Long lng_new = lng *10000000000L + Long lat_new = lat *10000000000L + udata.put("lng", lng_new) + udata.put("lat", lat_new) + if(lng2){ + Long lng2_new = lng2 *10000000000L + Long lat2_new = lat2 *10000000000L + udata.put("lng2", lng2_new) + udata.put("lat2", lat2_new) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata)) + } + + } + + }) + } + + def similarMatch(path){ + ExcelUtils.multiThreadingModify(path, { row -> + if(!row['F']){ + def d = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district") + if(d && d.lat){ + def dbo = getSimilarAmapLocation(d.city+"市", d.region, d.name, d.address, d.lat/10000000000.0, d.lng/10000000000.0) + if(dbo){ + row['F'] = dbo.region + row['G'] = dbo.name + row['H'] = dbo.address + row['I'] = formatAddress(d.city, dbo.address) + row['J'] = dbo.lng + row['K'] = dbo.lat + row['L'] = dbo.score + row['M'] = dbo.distance + row['S'] = dbo.type + } + } + } + + }) + } + + + + def getSimilarAmapLocation(city, region, name, address, lat, lng){ + + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "'") + sqlBuffer.append(" and MATCH(name,'${name}')") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + + def maxSocreObj = null + while (rs.next()) { + def map = [:] + def mapName = rs.getString("name").replace("·","") + + println name +" "+ mapName + if(rs.getString("adname").contains(region)){ + map.name = rs.getString("name") + map.address = rs.getString("address") + + map.region = rs.getString("adname") + map.type = rs.getString("type") + map.match_type = "名称相似" + def location = rs.getObject("location") + if(location){ + map.lng = location.x + map.lat = location.y + } + + double nameScore = 0 + double addScore = 0 + double loctionScore = 0 + def distance = com.fangjia.core.util.GeoUtils.calculateDistance(lng, lat, map.lng.toString().toDouble(), map.lat.toString().toDouble()); + map.distance = distance + if(map.name && name){ + nameScore = CosineSimilarAlgorithm.getSimilarity(map.name, name); + } + if(map.address && address){ + addScore = CosineSimilarAlgorithm.getSimilarity(map.address, address); + } + + if(hasColumn(rs, "shape")){ + def shape = rs.getObject("shape") + if(shape){ + def coordinates = shape.getObject("coordinates") +// def pointList = InsidePolygon.createPoint2(points); + def flag = InsidePolygon.isInArea(lng, lat, coordinates) + if(flag){ + loctionScore = 0.5 + } + } + }else{ + + if(distance < 500 && (nameScore>0.5 || addScore>0.5)){ + loctionScore = 0.3 + } + } + + def score = nameScore*0.3 + addScore*0.2 + loctionScore + map.score = score + if(!maxSocreObj){ + if(score>0){ + map.formatAddress = AddressUtil.formatAddress(city, map.address) + maxSocreObj = map + } + }else{ + println score +" "+maxSocreObj.score + if(maxSocreObj && score>maxSocreObj.score){ + map.formatAddress = AddressUtil.formatAddress(city, map.address) + maxSocreObj = map + } + } + } + } + return maxSocreObj; + } + + + def hasColumn(ResultSet rs, String columnName) { + ResultSetMetaData rsmd = rs.getMetaData(); + int columns = rsmd.getColumnCount(); + for (int x = 1; x <= columns; x++) { + if (columnName.equals(rsmd.getColumnName(x))) { + return true; + } + } + return false; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 464ac2301..75ff94eee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.lang.StringUtils import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -321,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{ def s = str.toString(); index++; if(s && s!="null"){ + s = s.replace('"','\\"') // if(s.toString().contains(",")){ // s = csvHandlerStr(s.toString()) // diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index dc389ffe5..644156419 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -60,7 +60,8 @@ class DistrictSellStatBatch extends BaseBatch { } def calcMonthlyPrice(){ - def citys = CommonUtils.findAllCity() + //只有福州使用tob价格 + def citys = CommonUtils.findAllCity() def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1) def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date())) @@ -142,17 +143,15 @@ class DistrictSellStatBatch extends BaseBatch { if(weekendPriceObj.value_type!=1){ if(dist.city == "福州"){ - if(Math.abs(mom)*100>3){ - if(dist.region != "马尾"){ - def randMarkup = nextDouble(0, 3) - if(mom < 0){ - mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) - }else{ - mom = DataUtil.roundDouble(randMarkup/100, 4) - } - curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) - } - } +// if(Math.abs(mom)*100>3){ +// def randMarkup = nextDouble(0, 3) +// if(mom < 0){ +// mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4) +// }else{ +// mom = DataUtil.roundDouble(randMarkup/100, 4) +// } +// curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price) +// } }else{ if(Math.abs(mom)*100>10){ def randMarkup = nextDouble(0, 10) @@ -232,6 +231,11 @@ class DistrictSellStatBatch extends BaseBatch { def dbcursor = DISTRICT_STAT.find(q) def priceMap = [:] for(def d : dbcursor){ + if(city == "福州"){ + if(d.avg_price_2b){ + d.avg_price = d.avg_price_2b + } + } priceMap.put(d.region +"##"+d.district_name, d) } return priceMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 2da506d11..917ddfc5e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -25,7 +25,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); -// task.regionPriceTrendRun() + task.regionPriceTrendRun() //// task.margeExcel() // task.saveCsvData() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 1dd114d8a..76925db82 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{ // sb.append(s+ "@|@") // } // } - sb.append(StringUtils.join(row.toArray(), "@|@")) + def str = StringUtils.join(row.toArray(), "@|@") + str = str.replace('"','\\"') + sb.append(str) + sb.append(System.getProperty("line.separator")); println index diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 9cea6cf74..aa277adde 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -2,6 +2,9 @@ package com.fangjia.script.fuzhou import com.fangjia.core.RegionData import com.fangjia.core.util.GeocodeUtils +import com.fangjia.core.util.JsonUtils +import com.fangjia.core.util.Message +import com.fangjia.core.util.MessageInfo import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsUtils import com.fangjia.script.util.DateUtils @@ -9,26 +12,71 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONObject import org.apache.commons.io.FileUtils import org.bson.types.ObjectId DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") +import com.fangjia.core.util.HttpRequest -ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> - def city = RegionData.formatCity("余姚") - d['D'] = city - if(city){ - d['E'] = RegionData.formatRegion(city, d['C']) - } -// if(!d['D']){ -// def citys = CommonDatautils.getCityByRegion(d['A']) -// if(citys && citys.size()==1){ -// d['F'] = citys.get(0) -// } +//Message message = new Message(); +//message.setMsgtype("text"); +//message.setText(new MessageInfo("测试")); +//URL url = new URL("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430"); +//// 建立http连接 +//HttpURLConnection conn = (HttpURLConnection) url.openConnection(); +//conn.setDoOutput(true); +//conn.setDoInput(true); +//conn.setUseCaches(false); +//conn.setRequestMethod("POST"); +//conn.setRequestProperty("Charset", "UTF-8"); +//conn.setRequestProperty("Content-Type", "application/Json; charset=UTF-8"); +//conn.connect(); +//OutputStream out = conn.getOutputStream(); +//String textMessage = JsonUtils.toJson(message); +//byte[] data = textMessage.getBytes("UTF-8"); +//out.write(data); +//out.flush(); +//out.close(); +//System.out.println(conn.getResponseCode()); +//InputStream ins = conn.getInputStream(); +//byte[] data1 = new byte[ins.available()]; +//ins.read(data1); +//System.out.println(new String(data1)); + +//HttpRequest request = new HttpRequest() +//def params = [:] +//params.put("","") +//request.post("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430","", "utf-8") + + +//sendMarkdownMessage("租金", "测试") +//static void sendMarkdownMessage(String title, String msg){ +// HttpRequest request = new HttpRequest() +// JSONObject requestBody = new JSONObject() +// JSONObject markdown = new JSONObject() +// markdown.put("title", title) +// markdown.put("text", msg) +// requestBody.put("markdown", markdown) +// requestBody.put("msgtype", "markdown") +// request.postWithBody("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430", requestBody, null, "UTF-8") +//} +//ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d -> +// def city = RegionData.formatCity("余姚") +// d['D'] = city +// if(city){ +// d['E'] = RegionData.formatRegion(city, d['C']) // } -}) +// +//// if(!d['D']){ +//// def citys = CommonDatautils.getCityByRegion(d['A']) +//// if(citys && citys.size()==1){ +//// d['F'] = citys.get(0) +//// } +//// } +//}) //def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1) //def m = [:] @@ -188,6 +236,24 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){ //},0) +def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据") +if(files){ + for(def filepath : files){ + def rets = read(filepath); + } +} + + +def read(path){ + File file = new File(path); + def rets = [] + file.eachLine{it-> + rets << it + } + return rets; +} + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index a3be166aa..63211976b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy index a57d7551b..34fa4eefe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddAvgpriceBatch.groovy @@ -29,10 +29,11 @@ class AddAvgpriceBatch { } def test(){ - def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"]) + def districts = CommonDatautils.findAllDistricts("福州","district", ["city","region","name", "estate_type2"]) +// def districts = CommonDatautils.findDistrictByCityAndRegions("福州","district", ["马尾"], ["city","region","name", "estate_type2"]) for(def dist : districts){ //发重算均价命令 - addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd')) + addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-06-30','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-07-05','yyyy-MM-dd')) } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index df4d9454e..e1e39ec3f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.linxsh.util.AddressUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import org.apache.commons.lang.StringUtils import java.util.regex.Matcher import java.util.regex.Pattern @@ -20,8 +21,10 @@ class FiledStrand { def updateIndex = 0; static main(args) { - def task = new FiledStrand(); - task.run(); + + +// def task = new FiledStrand(); +// task.run(); // task.updateOpenDate(); } @@ -265,13 +268,58 @@ class FiledStrand { static def processArea(all_area){ if(all_area){ def v = getValue(all_area) - if(v>=1000 && v<=10000000){ + if(v>=5000 && v<=10000000){ return v.toString().toDouble().toInteger() } } return null } + static def processStandardHeight(height){ + if(height){ + def v = getValue(height) + if(v>=2.4 && v<=10){ + return v.toString() + } + } + return null + } + + static def processElevators(elevators){ + if(elevators){ + def v = getValue(elevators) + if(v>0 && v<=100){ + return v.toString() + } + } + return null + } + + static def processStandardArea(area){ + if(area){ + area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","") + area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll(""); + return area + } + return null + } + + static def processCarSiteRent(rent){ + if(rent){ + def v = getNumberRange(rent) + if(v){ + if(v=="0"){ + return null + }else{ + return v.toString() + } + } + return null + } + return null + } + + static def processName(name){ name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期') if (name.contains('(') && name.contains(')')){ @@ -287,25 +335,25 @@ class FiledStrand { if (name.find('【')!=-1 && name.find('】')!=-1){ - def rep_str = name[name.index('【'):name.rfind('】')+1] + def rep_str = name.subsring(name.index('【'),name.rfind('】')+1) name = name.replace(rep_str, '') } if (name.find('【')!=-1 && name.find('】')!=-1){ - def rep_str = name[name.index('【'):name.rfind('】')+1] + def rep_str = name.subsring(name.index('【'),name.rfind('】')+1) name = name.replace(rep_str, '') } if (name.find('(')!=-1 && name.find(')')!=-1){ - def rep_str = name[name.index('('):name.rfind(')')+1] + def rep_str = name.subsring(name.index('('),name.rfind(')')+1) name = name.replace(rep_str, '') } if (name.find('「')!=-1 && name.find('」')!=-1){ - def rep_str = name[name.index('「'):name.rfind('」')+1] + def rep_str = name.subsring(name.index('「'),name.rfind('」')+1) name = name.replace(rep_str, '') } return name @@ -338,7 +386,7 @@ class FiledStrand { if(v){ if([40,50,70].contains(v)){ try { - return v.toString()+"年"; + return v.toString(); }catch(Exception e){ println e.getMessage() } @@ -378,6 +426,16 @@ class FiledStrand { return null } + static def processStandardAddress(def esAddress, def city){ + if(esAddress){ + def formatAddress = AddressUtil.formatAddress(city, esAddress) + if(formatAddress && isStandardRoad(formatAddress)){ + return formatAddress + } + } + return null + } + static def processBlock(city, region, def block){ if(block){ if(block.contains("无") || block.toString().length()>10){ @@ -464,6 +522,14 @@ class FiledStrand { return false } + static def isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } static def addBuildAreaToData(def dist, def updateFile){ @@ -629,7 +695,7 @@ class FiledStrand { println "estateCharege 转不了 "+estateCharege } } - return estateCharege; + return null; } static def processHouseholdCount(def household_count, def estateType2){ @@ -666,6 +732,34 @@ class FiledStrand { return null } + static def processFloor(floor){ + def v = getValue(floor.toString()); + if(v){ + if(v>=1 && v<=130){ + try { + return v.toString(); + }catch(Exception e){ + println e.getMessage() + } + } + } + return null + } + + static def processRealUsePercent(percent){ + def v = getValue(percent.toString()); + if(v){ + if(v>=20 && v<=80){ + try { + return v.toString(); + }catch(Exception e){ + println e.getMessage() + } + } + } + return null + } + static def processFinishBuildingDate(def finishBuildingDate){ if(finishBuildingDate){ Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}"); @@ -677,24 +771,37 @@ class FiledStrand { Pattern pat3 = Pattern.compile("\\d{4}"); def matcher3 = pat3.matcher(finishBuildingDate); + def dateStr = null if (matcher1.find()) { - return matcher1.group().replace("年","-").replace("月","-") + dateStr = matcher1.group().replace("年","-").replace("月","-").replace(".","") }else if(matcher2.find()){ - return matcher2.group().replace("年","-") + dateStr = matcher2.group().replace("年","-").replace(".","-") }else if(matcher3.find()){ - return matcher3.group() + dateStr = matcher3.group().replace(".","-") } + if(dateStr){ + dateStr = dateStr.substring(0, 4) + } + return dateStr } return null; } + + static def processCompany(val){ if(val){ if(val.length()>20){ return null }else{ - if(val.contains("无")){ + if(val.contains("暂无")){ + return null + } + if(val == "无"){ + return null + } + if(val.isNumber()){ return null } def fiterChars = [",",".","?","/","数据",'-','?',' ',' '] @@ -771,4 +878,31 @@ class FiledStrand { } return null; } + + static def patternParse(String keyword, String patternStr) { + Pattern pattern = Pattern.compile(patternStr); + Matcher matcher = pattern.matcher(keyword); + String num = ""; + if (matcher.find()) { + if (StringUtils.isNotBlank(matcher.group(0))) { + num = matcher.group(1).trim(); + } + } + return num; + } + + + + static def getNumberRange(def input) { + Pattern pattern = Pattern.compile("\\d+(-\\d+)?"); + Matcher matcher = pattern.matcher(input); + List list = new ArrayList(); + while (matcher.find()) { + list.add(matcher.group()); + } + if(list.size()>0){ + return list.get(0); + } + return null; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 16cae9383..4de90b96b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-04-01" + static def statDate = "2023-05-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy index 10efae15e..cc9c87682 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddDistrictElevatorBatch.groovy @@ -65,7 +65,7 @@ class AddDistrictElevatorBatch extends BaseBatch{ } def newHouses = analyElevator(haveHouses, dist.estate_type2) if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){ - println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 10层以上:"+newHouses.size() + println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size() BasicDBObject udata = new BasicDBObject() if(dist.estate_type2 == "别墅"){ udata.put("is_elevator", false) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index db058c5a7..a3236b53a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -1,9 +1,14 @@ package com.fangjia.script.gj.district import com.fangjia.core.util.HttpRequest +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.HttpRequestUtil +import com.mongodb.DBObject import net.sf.json.JSONArray import net.sf.json.JSONObject +import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang.StringUtils class FangjiaClientUtils { @@ -153,4 +158,18 @@ class FangjiaClientUtils { return columnValMap; } + + public static def saveOrUpdateDistrict(data) { + String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data) +// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr) +// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr) + + com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr) + if (json.status == 0) { + return "成功" + }else if(json.status == 1){ + return json.msg + } + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy index 9ec05ed28..af011e88f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -4,14 +4,25 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.FiledStrand +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import org.bson.types.ObjectId +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.regex.Pattern /** * 写字楼字段清洗入库(自动入库) @@ -19,6 +30,28 @@ import java.util.concurrent.Executors @BatchOwner(Watcher.GaoJuan) class OfficeFieldCleanBatch extends BaseBatch{ + //"build_date":"build_year", "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address", + // "live_date":"live_date", "loop_line_info":"loop_line" + def fieldsMap = ["estate_charge":"property_fee","block":"block","estate_company":"property_company","developer":"developer", + "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area" + ,"finish_building_date":"finish_building_date","property_life":"property_life","standard_area":"standard_area","total_floor":"total_floor", + "standard_height":"floor_height", "real_use_percent":"room_rate", "loop_line_info":"ring_location","car_site_count":"parking", + "parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area"]//key:seaweed表字段 value:es表字段 + + //"安居客":"安居客app", + def sourceMap = ["房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"] + + def fieldMap = ["编号":"_id","城市":"city","区域":"region","名称":"name","别名":"alias","租金":"rent_price","板块":"block","物业公司":"estate_company","物业费":"estate_charge", + "地址":"address","地址别名":"addr_alias","经度":"lng","纬度":"lat","类型":"estate_type2","建筑面积":"build_area","等级":"bed_kind", + "开发商":"developer","占地面积":"all_area","标准层面积":"standard_area","产权年限":"property_life","总层数":"total_floor","是否涉外":"is_foreign", + "容积率":"cubage_percent","绿化率":"virescence_percent","货梯数":"goods_elevator_count","竣工时间":"finish_building_date","客梯数":"person_elevators_count", + "标准层高":"standard_height","净高":"clear_height","大堂层高":"lobby_height","得房率":"real_use_percent","是否可分割":"divisible","环线位置":"loop_line_info", + "车位描述":"car_site","车位数量":"car_site_count","地上车位":"land_car_site","地下停车位":"underground_car_site","车位租金":"parking_fee","电梯数量":"elevators_count", + "是否注册":"is_register","开间面积":"bay_area","楼层信息":"floor_info"] + + static DBCollection BLOCKPOLYLINECOLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border2", "mongo.db.fangjia.com") + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static main(args) { @@ -27,12 +60,182 @@ class OfficeFieldCleanBatch extends BaseBatch{ @Override void process(Object args) { - def citys = CommonDatautils.findAllCity().reverse() - for(def c : citys){ - def offices = findAllOffice(c) - strandField(offices) +// def str = "1200平方/层,1067㎡,1650-1900平米" + + +// def citys = CommonDatautils.findAllCity().reverse() +// for(def c : citys){ +// def offices = findAllOffice(c) +// strandField(offices) +// } + +// addBlock() + +// addFieldValue() + +// updateErrorField() + export() + test() +// addOffice() + } + + def export(){ + def offices = CommonDatautils.findAllFiledDistrict("office", "上海") + def fields = fieldMap.keySet().asList() + def rets = [] + rets << fields + for(def i=0;i + def val = data.get(v) + if(k == "编号"){ + row << val.toString() + }else if(v=="lat" || v=="lng"){ + row << val.toString().toDouble()/10000000000.0 + }else if(v == "divisible"){ + + if(val==1){ + row << "是" + }else if(val==0){ + row << "否" + }else{ + row << '' + } + }else if(v.contains("alias")){ + if(val){ + row << val.join(" ") + }else{ + row << '' + } + }else{ + row << val + } + } + rets << row + } + } + ExcelUtils.write("D:\\数据\\写字楼\\中远.xlsx",rets, "数据") + } + + def test(){ + def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格租金.xlsx",0 , 1) + def map = [:] + for(def d : datas){ + map.put(d['A']+"#"+d['B']+'#'+d['C'], d) + } + ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\中远.xlsx",{d-> + def row = map.get(d['B']+"#"+d['C']+'#'+d['D']) + d['F'] = row['F'] + }) + } + + def updateErrorField(){ +// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"]) + def offices = CommonDatautils.findAllDistricts("上海","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"]) + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(offices.size()) + + for(def i=0;i40 || val < 0.01){ +//// println val +// udata.put("estate_charge", null) +//// } +// } + + if(udata){ + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',o._id),new BasicDBObject('$set',udata)) + } + } + + + def addFieldValue(){ + def citys = ["上海"] + def analysRets = [] + for(def c : citys){ + def totalCount = CommonDatautils.findAllDistrictCount("office", c) + if(totalCount){ + analysRets = addFieldFromMatch(c, totalCount, analysRets) + } + } } @@ -65,15 +268,431 @@ class OfficeFieldCleanBatch extends BaseBatch{ } - def findAllOffice(city){ - def sql = "select * from office where city='${city}'" - def rets = [] - ElasticUtils.executeSqlQuery(sql, { d-> - rets << d + def findEsDatas(c, esWebSource){ + def list = [] + ElasticUtils.executeSqlQuery("select * from office where city='${c}' and source='${esWebSource}'", {r-> + list << r + }) + return list + } + + + def matchDist(city, region, name){ + def dist = CommonDatautils.findDistrict(city, region, name, "office") + return dist; + } + + def matchApi(esRets){ + def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(esRets.size()) + for(def i=0;i=0.9){ + matchRets1 << r + } + } + return matchRets1 + } + + + def addFieldFromMatch(c, totalCount, analysRets){ + def fields = fieldsMap.keySet() as List + + sourceMap.each { seaweedmapSource, esWebSource -> + def esRets = findEsDatas(c, esWebSource) +// esRets = esRets[0..100] + def matchRets1 = matchApi(esRets) + def matchRets2 = selectRestByCredit(matchRets1) + println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size() + + def updateRets = processFieldsFromMatch(c , matchRets2) + println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+ DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%" + def precentMap = fieldUpdate(totalCount, updateRets) + println c +" "+precentMap.toString() + updateDB(c, updateRets) + println "------------------------------------------------------------------------------" + def pre1 = 0 + def pre2 = 0 + if(esRets){ + pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size()) + pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size()) + } + def analyRet = [] + analyRet << seaweedmapSource + analyRet << c + fields.each{key-> + analyRet << precentMap.get(key) + } + analyRet << esRets.size() + analyRet << matchRets1.size() + analyRet << matchRets2.size() + analyRet << pre1 + analyRet << pre2 + analysRets << analyRet + } + return analysRets; + } + + def fieldUpdate(total,updateRets){ + + def countMap = [:] + + for(def rm : updateRets){ + def valMap = rm.value + fieldsMap.each{seaweedfield,esField-> + if(valMap.containsKey(seaweedfield)) { + if(countMap.containsKey(seaweedfield)) { + countMap.put(seaweedfield, countMap.get(seaweedfield) + 1) + }else{ + countMap.put(seaweedfield, 1) + } + } + } + } + + println countMap.toString() + def precentMap = [:] + fieldsMap.each() {key,skey-> + if(countMap.containsKey(key)) { + precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total)) + }else { + precentMap.put(key, 0) + } + } + return precentMap; + } + + + def processFieldsFromMatch(c,matchRets){ + def fields = fieldsMap.keySet() as List + fields << "city" + fields << "estate_type2" + def fieldValueMap = [:] + for(def matchDist : matchRets){ + def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) + if(!dist){ + println c+" "+matchDist.match_id+" 不存在" + continue + } + def newValues = strandField(c, dist, matchDist) + if(newValues){ + println dist._id.toString()+" "+newValues.toString() + fieldValueMap.put(dist._id.toString(), newValues) + } + } + + return fieldValueMap; + } + + def strandField(c,dist, matchDist){ + def newValues = [:] + fieldsMap.each{seaweedfield,esField-> + def val = dist.get(seaweedfield) + if(!val){ + def esVal = matchDist.get(esField) + if(esVal){ + def newVal = null + if(seaweedfield == "estate_charge"){ + newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2) + newVal = newVal.toString() + } + if(seaweedfield == "build_date" || seaweedfield == "live_date"){ + newVal = FiledStrand.processFinishBuildingDate(esVal) + } + if(seaweedfield == "estate_company" || seaweedfield == "developer"){ + newVal = FiledStrand.processCompany(esVal) + } + if(seaweedfield =="virescence_percent"){ + newVal = FiledStrand.processVirescencePercent(esVal) + } + + if(seaweedfield =="cubage_percent"){ + newVal = FiledStrand.processCubagePercent(esVal) + } + + if(seaweedfield =="all_area" || seaweedfield =="build_area"){ + newVal = FiledStrand.processArea(esVal.toString()) + if(newVal){ + newVal = newVal.toString() + } + } + if(seaweedfield =="household_count"){ + newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2) + } + + if(seaweedfield =="loudong_count"){ + newVal = FiledStrand.processLoudongCount(esVal) + } + + if(seaweedfield =="car_site_count"){ + newVal = FiledStrand.processCarSite(esVal) + } + if(seaweedfield =="car_site"){ + newVal = FiledStrand.processCarSiteDesc(esVal) + } + + if(seaweedfield =="property_life"){ + newVal = FiledStrand.processPropertyLife(esVal) + } + + if(seaweedfield =="alias"){ + newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias) + } + + if(seaweedfield =="address"){ + newVal = FiledStrand.processAddress(esVal, dist.city) + } + if(seaweedfield =="block"){ + newVal = FiledStrand.processBlock(dist.city, dist.region, esVal) + } + + if(seaweedfield == "build_type"){ + newVal = FiledStrand.processBuildType(esVal, dist.estate_type2) + } + + if(seaweedfield == "land_property"){ + newVal = FiledStrand.processlandProperty(esVal) + } + + if(seaweedfield == "build_structure"){ + newVal = FiledStrand.processBuildStructure(esVal) + } + + if(seaweedfield == "total_floor"){ + newVal = FiledStrand.processFloor(esVal) + } + + if(seaweedfield == "real_use_percent"){ + newVal = FiledStrand.processRealUsePercent(esVal) + } + if(seaweedfield == "parking_fee"){ + newVal = FiledStrand.processCarSiteRent(esVal) + } + + if(seaweedfield == "elevators_count"){ + newVal = FiledStrand.processElevators(esVal) + } + + if(seaweedfield == "standard_area"){ + newVal = FiledStrand.processStandardArea(esVal) + } + + if(seaweedfield == "standard_height"){ + newVal = FiledStrand.processStandardHeight(esVal) + } + + if(seaweedfield == "bay_area"){ + newVal = esVal + } + + if(newVal){ + newValues.put(seaweedfield, newVal) + } + } + } + } + return newValues; + } + + def updateDB(c, updateRets){ + + for(def rm : updateRets) { + def id = rm.key + def valMap = rm.value + + BasicDBObject udata = new BasicDBObject() + valMap.each {k,v-> + udata.put(k,v) + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata)) + } + } + + def addBlock(){ + def rets = findAllBlockRange() + + def offices = CommonDatautils.findAllDistricts("上海", "office", ["city","region","name","block","lat2", "lng2"]) + + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(offices.size()) + + + for(def i=0;i<=offices.size();i++){ + final def office = offices[i] + if(office && !office.block){ + fixedThreadPool.execute(new Runnable() { + void run() { + try { + totalCount.addAndGet(1) + + def lastObj = null + for (def r : rets) { + def flag = InsidePolygon.isInArea(office.lng2 / 10000000000.0, office.lat2 / 10000000000.0, r.points) + if (flag && r.name) { + lastObj = r; + break; + } + } + if (lastObj) { + def udata = [:] + udata.put("block", lastObj.name) + udata.put("status", "0") + udata.put("cat", "office") + udata.put("username", "高娟") + udata.put("id", office._id.toString()) + udata.put("loopLineInfo", lastObj.loop) + def s = FangjiaClientUtils.saveOrUpdateDistrict(udata) + println s + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + } + + latch.await() + } + + def findAllBlockRange(){ + def rets = [] + def q = new BasicDBObject() + def dbcursor = BLOCKPOLYLINECOLL.find(q) + for(def d : dbcursor) { + def pointList = InsidePolygon.createPoint3(d.border); + d.points = pointList + rets << d + } + return rets; + } + + def saveOptLog(city, region, name, datas){ + def obj = new BasicDBObject(); + obj.put("city",city); + obj.put("region",region); + obj.put("district_name",name); + obj.put("category","office"); + obj.put("c_date", new Date()) + obj.put("m_date", new Date()) + obj.put("type", "update") + obj.put("data", datas) + obj.put("user_name", "高娟") + obj.put("origin", "script") + OPERATION_LOG_COLL.save(obj) + } + + def addOffice(){ + def city = "上海" + ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\bd_file_insert(待入库).xlsx",{d-> + def udata = [:] + def formatAddress = AddressUtil.formatAddress(city, d['B']) + udata.put("city", city) + udata.put("region", d['A']) + udata.put("name", d['D']) + udata.put("address", formatAddress) + long lng = (d['L'].toString().toDouble()*10000000000).longValue() + long lat = (d['K'].toString().toDouble()*10000000000).longValue() + + udata.put("lat", lat) + udata.put("lng", lng) + udata.put("status", "0") + udata.put("cat", "office") + udata.put("username", "高娟") + def s = FangjiaClientUtils.saveOrUpdateDistrict(udata) + println d['D'] + s }) - return rets } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy index 7107037b9..597926900 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SameDistName.groovy @@ -10,6 +10,9 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +/** + *处理地址在地址别名中有重复 + */ class SameDistName { static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") @@ -18,13 +21,13 @@ class SameDistName { def task = new SameDistName() // task.run() //地址和地址别名完全一致 -// task.addressPocess() + task.addressPocess() } def addressPocess(){ - def citys = CommonDatautils.findAllCity(); + def citys = ["福州"]//CommonDatautils.findAllCity(); for(def c : citys){ def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/house/RemoveHouse.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/house/RemoveHouse.groovy new file mode 100644 index 000000000..ec8b35d44 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/house/RemoveHouse.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj.house + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection + +@BatchOwner(Watcher.GaoJuan) +class RemoveHouse extends BaseBatch{ + + DBCollection SEABIRD_COLL = MongoUtils.getCollectionByName("fangjia", "seabird", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new RemoveHouse(), args) + } + + @Override + void process(Object args) { + def start = new Date() + def endDate = DateUtils.parseFromTightYYYYMMDD("2023-02-01") + def q = new BasicDBObject() + q.put("c_date", new BasicDBObject('$lte', endDate)) + def dbcursor = SEABIRD_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + + def rets = [] + for(def d : dbcursor){ + rets << d._id + if(rets.size()==200){ + def delquery = new BasicDBObject() + delquery.put("_id", new BasicDBObject('$in', rets)) + SEABIRD_COLL.remove(delquery) + rets = [] + print "." + } + } + def end = new Date() + + println "开始时间:"+start + println "结束时间:"+end + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy new file mode 100644 index 000000000..c92e387fe --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationBatch.groovy @@ -0,0 +1,121 @@ +package com.fangjia.script.gj.location + + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 判断小区是否在城市范围内 + */ +class DistrictLocationBatch { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + static DBCollection RANGECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline") + + static main(args) { + def task = new DistrictLocationBatch() +// task.run() + task.deleteDist() + } + + def deleteDist(){ + ExcelUtils.multiThreadingModify("d:\\数据\\小区\\非本城市小区.xlsx",{d-> + DBObject q = new BasicDBObject() + q.put("city", d['A']) + q.put("region", d['B']) + q.put("name", d['C']) + q.put("cat", 'district') + q.put("status", 0) + + def udata = new BasicDBObject() + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }) + } + + def run() { + def citys = CommonDatautils.findAllCity() + def cityMap = findCityRange() + def allrets = [] + for (def city : citys) { + def points = cityMap.get(city) + def pointList = InsidePolygon.createPoint3(points); + def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "lat", "lng"]) + def rets = findNotInCityDistricts(city, disticts, pointList) + allrets.addAll(rets) + } + ExcelUtils.write("d:\\数据\\小区\\非本城市小区.xlsx", allrets, "经纬度不在此城市中") + } + + def findNotInCityDistricts(city, districts, pointList){ + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + + for(def i=0;i + if(!citys1.contains(c)){ + c = c.replace("州","") + JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") + if(resp.containsKey("districts")){ + def datas = resp.get("districts") + for(def d : datas){ + println c +" "+ d.name+" "+d.level + if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){ + rows << d + }else{ + if(d.level=="city"){ + rows << d + } + } + } + } + } + } + + + +// citys.each{c-> +// println c +// JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8") +// if(resp.containsKey("districts")){ +// def datas = resp.get("districts") +// for(def d : datas){ +// println c +" "+ d.name+" "+d.level +// if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){ +// rows << d +// }else{ +// if(d.level=="city"){ +// rows << d +// } +// } +// } +// } +// } + if(rows){ + rows.each {r-> + def obj = new BasicDBObject() + obj.put("level", r.level) + obj.put("center", r.center) + obj.put("center", r.center) + obj.put("polyline", r.polyline) + obj.put("name", r.name) + obj.put("adcode", r.adcode) + obj.put("citycode", r.citycode) + + def city = RegionData.formatCity(r.name) + obj.put("fjw_city", city) + POLYLINECOLL.save(obj) + } + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy similarity index 80% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy index 3c2061a52..994019514 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionLocationUpdateBatch.groovy @@ -3,6 +3,9 @@ package com.fangjia.script.gj.location import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.GeocodeUtils +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils @@ -25,28 +28,122 @@ import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern; - -class LocationTest { +/** + * 根据区域围栏修正区域 + */ +@BatchOwner(Watcher.GaoJuan) +class RegionLocationUpdateBatch extends BaseBatch{ static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com") + - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { -// Date date=new Date(1622802609); -// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); -// String str = format.format(date); -// System.out.println(str); + runBatch(new RegionLocationUpdateBatch(), args); + } - def task = new LocationTest() - task.run() + @Override + public void process(Object args) { + regionUpdate() +// task.run() // task.run2() // task.run3() // task.run4() // task.run5() } + def regionUpdate(){ + def citys = CommonDatautils.findAllCity() + def rets = [] + def index = 0 + for(def city : citys){ + index ++ + def regionPointsMap = findRegionRange(city) + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]); + def datas = findInRegionDistricts(city, districts, regionPointsMap) + println "【${index}】"+city +" 经纬度处理完成" + rets.addAll(datas) + } + ExcelUtils.write("D:\\数据\\小区\\区域纠正.xlsx", rets, "数据") + } + + + def findInRegionDistricts(city, districts, regionPointsMap){ + def index = 0 + def rets = Collections.synchronizedList(new ArrayList()); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + def regionMap = [:] + regionPointsMap.each{k,v-> + def pointList = InsidePolygon.createPoint3(v); + regionMap.put(k, pointList) + } + def keys = regionPointsMap.keySet() + for(def i=0;i0){ def standardCity = RegionData.formatCity(cityMap.name) if(standardCity){ - saveData(standardCity, m.name, m.ad_code , ret); + def newRegion = RegionData.formatRegion(standardCity, m.name) + saveData(pro, standardCity, m.name, m.ad_code , ret, newRegion); }else{ println cityMap.name +" 没有标准化" } @@ -102,7 +103,7 @@ class RegionPointCrawl { def findCityCode(city){ Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE); def q = null; - if(city.contains("省") || city.endsWith("自治区")){ + if(city.contains("省") || city.endsWith("自治区") || ["上海市","天津市","北京市","重庆市"].contains(city)){ q = new BasicDBObject("province", pattern); }else{ q = new BasicDBObject("name", pattern); @@ -156,18 +157,22 @@ class RegionPointCrawl { } } - def saveData(city, region, code, ret){ + def saveData(pro, city, region, code, ret, formatRegion){ def q = new BasicDBObject() q.put("code", code) def dbo = POLYLINECOLL.findOne(q) if(!dbo){ def o = new BasicDBObject() + o.put("province", pro) o.put("city", city) o.put("region", region) o.put("code", code) o.put("center", ret.center.get(0)) o.put("level", ret.level.get(0)) o.put("polyline", ret.polyline.get(0)); + if(formatRegion){ + o.put("fjw_region", formatRegion) + } POLYLINECOLL.save(o); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 1068d1f84..788d1703e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") //广州,佛山 重庆 @@ -204,8 +204,11 @@ public class BeiKeCrawlByChrome { if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl def dbo = findTrade(durl , "贝壳找房") + if (!dbo) { - parse(city, e, region, driver) + parse(city, e, region, driver, null) + }else if(dbo && !dbo.lift){ + parse(city, e, region, driver, dbo._id) } }else{ stop = true; @@ -224,7 +227,7 @@ public class BeiKeCrawlByChrome { return detailUrls } - def parse(city,e, region, driver){ + def parse(city,e, region, driver, id){ try { def detailUrl = e.select(".title a").attr("href") @@ -289,20 +292,26 @@ public class BeiKeCrawlByChrome { } detail.source = "贝壳找房" - def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc) + def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc) + + def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","") if(liftStr){ detail.lift = DataFormatUtils.parseLift(liftStr) detail.lift_household = DataFormatUtils.parseHousehold(liftStr) } - detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") - detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) - detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) - detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) - detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) - detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) - detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) - detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) - def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡","")) + detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构","")) + detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡","")) + + detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","") + detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属","")) + detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途","")) + detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途","")) + detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限","")) if(detail.total_price && detail.avg_price){ detail.url = detailUrl @@ -323,8 +332,11 @@ public class BeiKeCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail - DEAL_PRICE_COLL.save(detail) - + if(id){ + DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail)) + }else{ + DEAL_PRICE_COLL.save(detail) + } } } }catch(Exception ex){ @@ -453,9 +465,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = "https:"+e.attr("href") -// if (citys.contains(e.text())){ + if (citys.contains(e.text())){ list << [e.text(), href] -// } + } } } return list @@ -483,7 +495,7 @@ public class BeiKeCrawlByChrome { elemPassword.sendKeys("12qwaszx"); btn.submit(); // 点击行为; - Thread.sleep(20000) + Thread.sleep(10000) }catch(Exception e){ Thread.sleep(15000) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0c82ca47e..6c0ce91f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.paimai import com.fangjia.core.RegionData +import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.gj.ccb.FileUtils @@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome { def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) - def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM") static main(args) { def task = new LianjiaCrawlByChrome(); @@ -297,16 +298,23 @@ public class LianjiaCrawlByChrome { def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd") - if(dealDate.getTime()>=endDate.getTime()){ - detailUrls << durl - def dbo = findTrade(durl , "链家在线") - if (!dbo) { - parse(city, e, regionText, driver) + if(dealDate){ + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + def dbo = findTrade(durl , "链家在线") + if (!dbo) { + parse(city, e, regionText, driver, null) + }else if(dbo && !dbo.lift){ + parse(city, e, regionText, driver, dbo._id) + } + }else{ + stop = true; + break; } }else{ - stop = true; - break; + println url +" 成交时间有误!" } + } if(page>=100){ stop = true; @@ -320,14 +328,13 @@ public class LianjiaCrawlByChrome { return detailUrls } - def parse(city,e, regionText, driver){ + def parse(city,e, regionText, driver, id){ try { def price = e.select(".totalPrice .number").text() if(price){ def detailUrl = e.select(".title a").attr("href") driver.get(detailUrl) def detailDoc = checkDocument(driver, detailUrl); - def title = e.select(".title a").text() if (title) { def detail = new BasicDBObject() @@ -345,6 +352,15 @@ public class LianjiaCrawlByChrome { if(detail.region){ detail.region = RegionData.formatRegion(city, detail.region) } + if(!detail.region){ + def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1) + str = str.substring(0, str.indexOf("',")) + def locArrs = str.split(",") + def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble()) + if(gmap){ + detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat) + } + } def roomStr = arrs[1] if (roomStr) { @@ -386,20 +402,22 @@ public class LianjiaCrawlByChrome { detail.source = "链家在线" detail.url = detailUrl - def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","") + def detailDiv = JQueryUtils.select(".introContent", detailDoc) + + def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","") if(liftStr){ detail.lift = DataFormatUtils.parseLift(liftStr) detail.lift_household = DataFormatUtils.parseHousehold(liftStr) } - detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","") - detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构","")) - detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属","")) - detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限","")) - detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯","")) - detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构","")) - detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代","")) - detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型","")) - def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层","")) + detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","") + detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构","")) + detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属","")) + detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限","")) + detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯","")) + detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构","")) + detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代","")) + detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型","")) + def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层","")) if(detail.total_price && detail.avg_price){ def houseInfo = e.select(".houseInfo").text() @@ -417,7 +435,12 @@ public class LianjiaCrawlByChrome { detail.create_date = new Date() detail.m_date = new Date() println detail - DEAL_PRICE_COLL.save(detail) + if(id){ + DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail)) + }else{ + DEAL_PRICE_COLL.save(detail) + } + } } } @@ -547,9 +570,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") -// if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia") && citys.contains(e.text())){ cityList << [e.text(), href] -// } + } } } return cityList @@ -569,9 +592,11 @@ public class LianjiaCrawlByChrome { WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - elemUsername.sendKeys("15021384995"); +// elemUsername.sendKeys("15021384995"); +// elemPassword.sendKeys("12qwaszx"); - elemPassword.sendKeys("12qwaszx"); + elemUsername.sendKeys("16283166371"); + elemPassword.sendKeys("Goojia123456"); btn.submit(); // 点击行为; Thread.sleep(15000) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index f419acf6d..67c803131 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -21,10 +22,11 @@ class AddPriceTest { task.run(); } def run(){ - def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx" + def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx" addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) +// addFjwPrice(filePath) } @@ -105,12 +107,12 @@ class AddPriceTest { def addGXDPrice(filepath){ ExcelUtils.multiThreadingModify(filepath, { d -> if (d['B']) { -// def dist = CommonDatautils.findDistrictById(d['B']) -// if(dist){ -// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) - def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) - d['G'] = gxdPrice -// } + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ + def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name) +// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C']) + d['I'] = gxdPrice + } } }) } @@ -129,4 +131,22 @@ class AddPriceTest { } return price } + + + def addFjwPrice(path){ + ExcelUtils.multiThreadingModify(path,{d-> + if(!d['K']){ + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ +// d['N'] = dist.region + def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd")) + if(dbo){ +// d['K'] = dbo.avg_price + d['K'] = dbo.avg_price_2b + } + } + } + }) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy index bb19fe69d..72df7724e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/DayUpdateDistRefreshBatch.groovy @@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{ if(dists){ for(def d : dists){ - def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) + def distObj = null + if(d.comm_id){ + distObj = CommonDatautils.findDistrictById(d.comm_id) + }else{ + distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields) + } if(distObj && distObj._id){ - FzDataUtils.converDistrict(distObj, null, null); + if(d.type && d.type==1){ + FzDataUtils.removeById(d.comm_id) + }else{ + FzDataUtils.converDistrict(distObj, null, null); + } println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name updateTaskStatus(d.id, 2); } @@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{ def q = new BasicDBObject() q.put("status", 5);//未处理 TASK_COLL.find(q).each {d-> - rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name] + rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id] } return rets; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy index bf6e2c177..51e58e28e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/FzDataUtils.groovy @@ -119,6 +119,13 @@ class FzDataUtils { } } + + static def removeById(id){ + def del_data = new BasicDBObject() + del_data.put("num_id", id) + DIST_COLL.remove(del_data) + } + static def saveDistrict(def m){ DBObject q = new BasicDBObject(); q.put("num_id",m.num_id) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy index 517b6b634..61cc61f88 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy @@ -294,4 +294,8 @@ class AddressUtil { return result } + public static void main(String[] args){ + def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区") + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy index fd646bf20..03f3e2b73 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/looplineinfo/InsidePolygon.groovy @@ -52,7 +52,40 @@ class InsidePolygon { return list; } - + + + + public static List createPoint3(String points){ + + List alllist = new ArrayList(); + if(points){ + List arrs = new ArrayList(); + if(points.contains("|")){ + String[] array = points.split("\\|"); + arrs.addAll(array) + }else{ + arrs.add(points) + } + + for(def arrStr : arrs){ + List list = new ArrayList(); + String[] array = arrStr.split(";"); + for(int i =0 ;i> getPoints(String points){ String[] array = points.split("-"); List> list = new ArrayList>(); @@ -227,4 +260,16 @@ class InsidePolygon { } return false; } + + + public static def isInArea(lng, lat, pointList){ + def flag = false; + for(def pointArr : pointList){ + flag = InsidePolygon.isPtInPoly(lng, lat, pointArr) + if(flag){ + break; + } + } + return flag; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index c1a7d36eb..db1f5a13d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -24,7 +24,7 @@ class CommitteeLatLng { static ExecutorService executorService = Executors.newFixedThreadPool(10) static void main(String[] args) { - dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021)) + dealPoi(new BasicDBObject("location", "").append("year", 2022)) //dealLocation() executorService.shutdown() } @@ -71,7 +71,7 @@ class CommitteeLatLng { if (amapPoi && amapPoi.size() > 0) { def poi = amapPoi[0] double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, name) - if (similarity <= 0.8) { + if (similarity <= 0.8 && !String.valueOf(poi.name).contains(name)) { amapPoi = GaoDeUtils.searchPoi(city, address, area, "") } poi = amapPoi[0] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 2a34dd4b3..30b2e3a65 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -55,13 +55,17 @@ class GaoDeUtils { * @return */ static String getGaodeKey(String type) { - Map params = new HashMap<>(); - params.put("type", type) - JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params) - if (json.status == 0 && json.result != null) { - return json.result - } - System.exit(1); +// Map params = new HashMap<>(); +// params.put("type", type) +// JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params) +// if (json.status == 0 && json.result != null) { +// return json.result +// } +// System.exit(1); + return "8796b9dfb3889b57c38e2fff1b7e51bd"; +// 'eb69e4ad2a9cdd31f163a0ed32effe6e', '60f764d228d64addf390e28d68a01a35', '66e6771767ab293ace441553c36621c1', +// 'a63254b4a1b2e6f4bb11058e0b5d9fa2', '743ad872ad39f0a10665b28fa968af0b', '3b7c1d201c403bb416b63535ed3d0e98', +// 'a34894d151fffff94f9a33bae3d5565c', '389e0a3a155e8d28728c3faede4ebd4f' } /** @@ -330,10 +334,10 @@ class GaoDeUtils { params.page = "${page}".toString() page++ params.key = this.getGaodeKey("v5/place/text") - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); if (json == null) { params.key = this.getGaodeKey("v5/place/text") - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy index ff11d6991..69c889afe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.temp.shenjian.util +import net.sf.json.JSONObject import org.apache.commons.httpclient.HttpClient import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.RequestEntity import org.apache.commons.httpclient.methods.StringRequestEntity /** @@ -26,4 +28,42 @@ class HttpUtils { } return false } + + + static def doPost2(String url, String jsonString){ + HttpClient httpClient = new HttpClient() + PostMethod postMethod = new PostMethod(url) + String responseMsg = null; + try{ + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + postMethod.setRequestEntity(requestEntity); + httpClient.executeMethod(postMethod); + InputStream is = postMethod.getResponseBodyAsStream(); + BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8")); + StringBuffer sb = new StringBuffer(""); + String str= ""; + while((str = br.readLine()) != null){ + sb.append(str); + } + responseMsg = sb.toString(); + } catch (Exception ex) { + ex.printStackTrace(); + } finally { + postMethod.releaseConnection(); + } + return JSONObject.fromObject(responseMsg); + } + + public String encodeHtml(String html) { + String mh=html; + try { + mh=new String(html.getBytes("iso-8859-1")); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + return mh; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index b5f86bf1b..8379df446 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -92,8 +92,10 @@ class DataStandardBatch extends BaseBatch{ cityMap.each{city,alias-> println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) - districtMap.putAll(detailMap) +// if(city == "广州"){ + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"]) + districtMap.putAll(detailMap) +// } } while(flag){ @@ -833,7 +835,7 @@ class DataStandardBatch extends BaseBatch{ } } } - if(!dbo){ + if(!dbo && isStandardRoad(copyObj.address)){ result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address); if(result.size()==1){ dbo = result[0]; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index 320e65b2f..4fe5dd4a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{ } @Override void process(Object args) { - println SHORT_DATE + def citys = CommonDatautils.findAllCity(); def weekend = DateUtils.getFirstDayOfWeek(new Date()); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy index c68ba4a85..67858c081 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy @@ -28,7 +28,25 @@ class TradeDealPriceStat { static main(args) { def task = new TradeDealPriceStat() - task.run() +// task.run() + task.deleteRepeat() + } + + def deleteRepeat(){ + def q = new BasicDBObject() + q.put("source", "我爱我家") + def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT) + def set = new HashSet() + for(def c : cursor){ + def tradeDate = DateUtils.formatDateByPattern(c.trade_date, "yyyyMMdd") + def key = c.city +"#"+c.region+"#"+c.district_name+"#"+c.avg_price+"#"+tradeDate + if(set.contains(key)){ + DEAL_PRICE.remove(new BasicDBObject('_id', c._id)) + println c._id + }else{ + set << key + } + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy index b1a045e3f..520cf9f99 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/DongGuanBank.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20230301", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230401", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -38,6 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303") +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index bbc24f035..7b199ce7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220901", "yyyyMMdd") -def endDate = DateUtil.parseDate("20221001", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230201", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230301", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java b/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java new file mode 100644 index 000000000..58f49d8da --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/bo/BuildingRatioBo.java @@ -0,0 +1,102 @@ +package com.fangjia.service.bo; + +import java.io.Serializable; +import java.util.Date; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioBo implements Serializable { + + private static final long serialVersionUID = 1L; + private String id; + private String city_name; + private String region; + private String district; + private String building_name; + private Double ratio; + private Date c_date; + private Date m_date; + private String create_user; + private String note; + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCity_name() { + return city_name; + } + + public void setCity_name(String city_name) { + this.city_name = city_name; + } + + public String getRegion() { + return region; + } + + public void setRegion(String region) { + this.region = region; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getBuilding_name() { + return building_name; + } + + public void setBuilding_name(String building_name) { + this.building_name = building_name; + } + + public Double getRatio() { + return ratio; + } + + public void setRatio(Double ratio) { + this.ratio = ratio; + } + + public Date getC_date() { + return c_date; + } + + public void setC_date(Date c_date) { + this.c_date = c_date; + } + + public Date getM_date() { + return m_date; + } + + public void setM_date(Date m_date) { + this.m_date = m_date; + } + + public String getCreate_user() { + return create_user; + } + + public void setCreate_user(String create_user) { + this.create_user = create_user; + } + + public String getNote() { + return note; + } + + public void setNote(String note) { + this.note = note; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioService.java new file mode 100644 index 000000000..36a450706 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioService.java @@ -0,0 +1,18 @@ +package com.fangjia.service.mongo.bkoff; + +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; + +import java.util.List; +import java.util.Map; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public interface BuildingRatioService { + public void save(BuildingRatio buildingRatio); + public void deleteById(String id); + public void saveOrUpdateByBuildingName(String city, String region, String district, String building_name, Map data); + public BuildingRatio findBuildingRatioById(String id); + public List findAllByBuildingName(String city, String region, String district, String building_name); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioServiceImpl.java new file mode 100644 index 000000000..bee82f5dd --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/bkoff/BuildingRatioServiceImpl.java @@ -0,0 +1,81 @@ +package com.fangjia.service.mongo.bkoff; + +import com.fangjia.dal.QueryParam; +import com.fangjia.dal.SortParam; +import com.fangjia.dal.mongo.dao.bkoff.BuildingRatioDao; +import com.fangjia.dal.mongo.model.bkoff.BuildingRatio; +import com.fangjia.dal.sql.model.FjtUsers; +import org.apache.commons.lang.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @author zhaojixiang + * @since 2023/7/5 + */ +public class BuildingRatioServiceImpl implements BuildingRatioService{ + + private BuildingRatioDao buildingRatioDao; + + @Override + public void save(BuildingRatio buildingRatio) { + + this.buildingRatioDao.save(buildingRatio); + } + + @Override + public void deleteById(String id) { + this.buildingRatioDao.deleteById(id); + } + @Override + public void saveOrUpdateByBuildingName(String city, String region, String district, String building_name, Map data) { + List params = new ArrayList(); + params.add(new QueryParam("city", city)); + params.add(new QueryParam("region", region)); + params.add(new QueryParam("district", district)); + params.add(new QueryParam("building_name", building_name)); + data.put("city", city); + data.put("region", region); + data.put("district", district); + data.put("building_name", building_name); + this.buildingRatioDao.updateByParams(data, params, true); + } + + @Override + public BuildingRatio findBuildingRatioById(String id) { + + return this.buildingRatioDao.findById(id); + } + + @Override + public List findAllByBuildingName(String city, String region, String district, String building_name) { + List params = new ArrayList(); + if(StringUtils.isNotBlank(city)){ + params.add(new QueryParam("city", city)); + } + if(StringUtils.isNotBlank(region)){ + params.add(new QueryParam("region", region)); + } + if(StringUtils.isNotBlank(district)){ + params.add(new QueryParam("district", district)); + } + if(StringUtils.isNotBlank(building_name)){ + params.add(new QueryParam("building_name", building_name)); + } + + List sortParams = new ArrayList(); + sortParams.add(new SortParam("m_date", false)); + + return this.buildingRatioDao.findAllByParams(params, sortParams); + } + + public BuildingRatioDao getBuildingRatioDao() { + return buildingRatioDao; + } + + public void setBuildingRatioDao(BuildingRatioDao buildingRatioDao) { + this.buildingRatioDao = buildingRatioDao; + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/open/ApiUserBo.java b/projects/fangjia-service/src/main/java/com/fangjia/service/open/ApiUserBo.java index 4eae77e54..8c09268bf 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/open/ApiUserBo.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/open/ApiUserBo.java @@ -26,6 +26,7 @@ public class ApiUserBo implements Serializable { private Date deadTime; private String manager; private String priceType; + private Integer isMonitor; public String getPriceType() { return priceType; @@ -162,4 +163,12 @@ public class ApiUserBo implements Serializable { public void setManager(String manager) { this.manager = manager; } + + public Integer getIsMonitor() { + return isMonitor; + } + + public void setIsMonitor(Integer isMonitor) { + this.isMonitor = isMonitor; + } }