This commit is contained in:
2023-07-14 14:28:48 +08:00
71 changed files with 3764 additions and 268 deletions
@@ -102,6 +102,7 @@ public class ApiUserController {
bo.setPurpose(apiUser.getPurpose());
bo.setDeadTime(apiUser.getDeadTime());
bo.setManager(apiUser.getManager());
bo.setIsMonitor(apiUser.getIsMonitor());
apiUserBoList.add(bo);
}
@@ -183,6 +184,7 @@ public class ApiUserController {
apiUser.setModificationDate(date);
apiUser.setDeadTime(model.getDeadTime());
apiUser.setManager(model.getManager());
apiUser.setIsMonitor(model.getIsMonitor());
this.apiUserService.save(apiUser);
if(model.getEvaluateRate() != null && model.getEvaluateRate() != 0){
ApiUserCfg apiUserCfg = new ApiUserCfg();
@@ -222,6 +224,7 @@ public class ApiUserController {
bo.setEmail(apiUser.getEmail());
bo.setDeadTime(apiUser.getDeadTime());
bo.setManager(apiUser.getManager());
bo.setIsMonitor(apiUser.getIsMonitor());
ApiUserCfg apiUserCfg_rate = apiUserCfgService.findByUserId(apiUser.getId(), ApiUserCfg.ApiUserCfgKey.EVALUATE_RATE);
if(apiUserCfg_rate != null){
bo.setEvaluateRate((Double) apiUserCfg_rate.getCfgValue());
@@ -272,6 +275,7 @@ public class ApiUserController {
dataApiUser.put("purpose", model.getPurpose());
dataApiUser.put("deadTime", model.getDeadTime());
dataApiUser.put("manager", model.getManager());
dataApiUser.put("isMonitor", model.getIsMonitor());
this.apiUserService.update(dataApiUser, userId);
//ApiUserCfg.ApiUserCfgKey.EVALUATE_RATE
@@ -0,0 +1,167 @@
package com.fangjia.bkoff.controllers;
import com.fangjia.bkoff.model.ApiUserModel;
import com.fangjia.bkoff.model.BuildingRatioModel;
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
import com.fangjia.dal.sql.model.User;
import com.fangjia.service.bo.BuildingRatioBo;
import com.fangjia.service.mongo.bkoff.BuildingRatioService;
import org.apache.commons.lang.StringUtils;
import java.net.URLEncoder;
import java.util.*;
/**
* @author zhaojixiang
* @since 2023/7/5
*/
public class BuildingRatioController {
private BuildingRatioService buildingRatioService;
/**
* 新建楼栋系数信息
*
* @param model
* @return
*/
public Object doAddAction(BuildingRatioModel model) throws Exception {
try {
if(checkBuildingNameAction(model)){
model.redirect("/buildingRatio/add?msg=" + URLEncoder.encode(model.getMsg(), "UTF-8"));
return model;
}
Date date = new Date();
BuildingRatio buildingRatio = new BuildingRatio();
buildingRatio.setCity(model.getCity_name());
buildingRatio.setRegion(model.getRegion());
buildingRatio.setDistrict(model.getDistrict());
buildingRatio.setBuilding_name(model.getBuilding_name());
buildingRatio.setRatio(model.getRatio());
buildingRatio.setC_date(date);
buildingRatio.setM_date(date);
buildingRatio.setCreate_user(model.getCreate_user());
buildingRatio.setNote(model.getNote());
this.buildingRatioService.save(buildingRatio);
model.setMsg("新建成功!");
} catch (Exception e) {
e.printStackTrace();
model.setMsg("新建失败!");
}
model.redirect("/buildingRatio/add?msg=" + URLEncoder.encode(model.getMsg(), "UTF-8"));
return model;
}
public Object doDeleteAction(BuildingRatioModel model){
if(StringUtils.isNotBlank(model.getId())){
this.buildingRatioService.deleteById(model.getId());
}
model.setViewType("redirect");
model.setViewParam("location","/buildingRatio/list");
return model;
}
/**
* 修改楼栋系数信息
*
* @param model
* @return
*/
public Object updateAction(BuildingRatioModel model) throws Exception {
Map<String, Object> data_buildingRatio = new HashMap<String, Object>();
data_buildingRatio.put("id", model.getId());
data_buildingRatio.put("city", model.getCity_name());
data_buildingRatio.put("region", model.getRegion());
data_buildingRatio.put("district", model.getDistrict());
data_buildingRatio.put("building_name", model.getBuilding_name());
data_buildingRatio.put("ratio", model.getRatio());
data_buildingRatio.put("m_date", new Date());
data_buildingRatio.put("create_user", model.getCreate_user());
data_buildingRatio.put("note", model.getNote());
this.buildingRatioService.saveOrUpdateByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name(), data_buildingRatio);
model.setViewType("redirect");
model.setViewParam("location", "/buildingRatio/list");
return model;
}
public Object listAction(BuildingRatioModel model) {
List<BuildingRatio> buildingRatioList = this.buildingRatioService.findAllByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name());
List<BuildingRatioBo> buildingRatioBoList = new ArrayList<BuildingRatioBo>();
for (BuildingRatio buildingRatio : buildingRatioList) {
BuildingRatioBo bo = new BuildingRatioBo();
bo.setId(buildingRatio.getId());
bo.setCity_name(buildingRatio.getCity());
bo.setRegion(buildingRatio.getRegion());
bo.setDistrict(buildingRatio.getDistrict());
bo.setBuilding_name(buildingRatio.getBuilding_name());
bo.setRatio(buildingRatio.getRatio());
bo.setC_date(buildingRatio.getC_date());
bo.setM_date(buildingRatio.getM_date());
bo.setCreate_user(buildingRatio.getCreate_user());
bo.setNote(buildingRatio.getNote());
buildingRatioBoList.add(bo);
}
model.setBuildingRatioBoList(buildingRatioBoList);
return model;
}
public Object addAction(BuildingRatioModel model) {
return model;
}
/**
* 编辑api用户信息
*
* @param model
* @return
*/
public Object editAction(BuildingRatioModel model) {
BuildingRatio buildingRatio = this.buildingRatioService.findBuildingRatioById(model.getId());
BuildingRatioBo bo = new BuildingRatioBo();
bo.setId(buildingRatio.getId());
bo.setCity_name(buildingRatio.getCity());
bo.setRegion(buildingRatio.getRegion());
bo.setDistrict(buildingRatio.getDistrict());
bo.setBuilding_name(buildingRatio.getBuilding_name());
bo.setRatio(buildingRatio.getRatio());
bo.setCreate_user(buildingRatio.getCreate_user());
bo.setNote(buildingRatio.getNote());
model.setBuildingRatioBo(bo);
if (StringUtils.isNotBlank(model.getMsg()) && model.getMsg().equals("success")) {
model.setMsg("更新成功!");
}
return model;
}
public boolean checkBuildingNameAction(BuildingRatioModel model) {
model.setViewType("json");
model.setViewParam("config", null);
List<BuildingRatio> buildingRatioList = buildingRatioService.findAllByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name());
if (buildingRatioList.size()>0) {
model.setMsg("此楼栋名称已经存在!");
return true;
}
return false;
}
public BuildingRatioService getBuildingRatioService() {
return buildingRatioService;
}
public void setBuildingRatioService(BuildingRatioService buildingRatioService) {
this.buildingRatioService = buildingRatioService;
}
}
@@ -38,6 +38,7 @@ public class ApiUserModel extends BaseModel {
private Integer creditLevelStatus;
private String priceType;
private Map<String,Double> priceRatingMap;
private Integer isMonitor;
public String getPriceType() {
return priceType;
@@ -606,4 +607,12 @@ public class ApiUserModel extends BaseModel {
public void setManager(String manager) {
this.manager = manager;
}
public Integer getIsMonitor() {
return isMonitor;
}
public void setIsMonitor(Integer isMonitor) {
this.isMonitor = isMonitor;
}
}
@@ -0,0 +1,134 @@
package com.fangjia.bkoff.model;
import com.fangjia.service.bo.BuildingRatioBo;
import org.springframework.format.annotation.DateTimeFormat;
import java.util.Date;
import java.util.List;
/**
* @author zhaojixiang
* @since 2023/7/5
*/
public class BuildingRatioModel extends BaseModel{
private String id;
private String city_name;
private String region;
private String district;
private String building_name;
private String create_user;
private String note;
private Double ratio;
private String msg;
private List<BuildingRatioBo> buildingRatioBoList;
private BuildingRatioBo buildingRatioBo;
@DateTimeFormat(pattern = "yyyy-MM-dd")
private Date c_date;
@DateTimeFormat(pattern = "yyyy-MM-dd")
private Date m_date;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getCity_name() {
return city_name;
}
public void setCity_name(String city_name) {
this.city_name = city_name;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district_name) {
this.district = district_name;
}
public String getBuilding_name() {
return building_name;
}
public void setBuilding_name(String building_name) {
this.building_name = building_name;
}
public Double getRatio() {
return ratio;
}
public void setRatio(Double ratio) {
this.ratio = ratio;
}
public Date getC_date() {
return c_date;
}
public void setC_date(Date c_date) {
this.c_date = c_date;
}
public Date getM_date() {
return m_date;
}
public void setM_date(Date m_date) {
this.m_date = m_date;
}
public List<BuildingRatioBo> getBuildingRatioBoList() {
return buildingRatioBoList;
}
public void setBuildingRatioBoList(List<BuildingRatioBo> buildingRatioBoList) {
this.buildingRatioBoList = buildingRatioBoList;
}
public BuildingRatioBo getBuildingRatioBo() {
return buildingRatioBo;
}
public void setBuildingRatioBo(BuildingRatioBo buildingRatioBo) {
this.buildingRatioBo = buildingRatioBo;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
public String getCreate_user() {
return create_user;
}
public void setCreate_user(String create_user) {
this.create_user = create_user;
}
public String getNote() {
return note;
}
public void setNote(String note) {
this.note = note;
}
}
@@ -57,13 +57,20 @@
<td>账户用途</td>
<td>
<select name="purpose">
<option value="" >请选择</option>
<option value="" >请选择</option>
<option value="1" >API</option>
<option value="2" >PDS</option>
<option value="4" >房生活</option>
</select>
</td>
</tr>
<tr>
<td>是否监控</td>
<td>
<input type="radio" name="isMonitor" value="0" checked="checked"/>不监控
<input type="radio" name="isMonitor" value="1" style="margin-left: 15px;"/>监控
</td>
</tr>
<tr>
<td>Email</td>
<td><input name="email" value="${email!}" /></td>
@@ -72,6 +72,13 @@
</select>
</td>
</tr>
<tr>
<td>是否监控</td>
<td>
<input type="radio" <#if apiUserBo?? && apiUserBo.isMonitor?? && apiUserBo.isMonitor == 0>checked="checked"</#if> name="isMonitor" value="0" />不监控
<input type="radio" <#if apiUserBo?? && apiUserBo.isMonitor?? && apiUserBo.isMonitor == 1>checked="checked"</#if> name="isMonitor" value="1" style="margin-left: 15px;"/>监控
</td>
</tr>
<tr>
<td>Email</td>
<td><input name="email" value="${apiUserBo.email!}"/></td>
@@ -53,6 +53,9 @@
<th align="center" width="60px">
账户用途
</th>
<th align="center" width="60px">
是否监控
</th>
<th align="center" width="120px">
到期时间
</th>
@@ -114,6 +117,13 @@
房生活
</#if>
</td>
<td>
<#if bo.isMonitor==1>
是
<#else>
否
</#if>
</td>
<td align="center">
<#if bo.deadTime??>
${bo.deadTime?string("yyyy年MM月dd日")}
@@ -0,0 +1,112 @@
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
<#compress>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>楼栋系数新增</title>
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
<#include "/common_js.ftl" />
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
</head>
<body>
<#include "/layout/header.ftl" />
<div class="main-content">
<font color="red">${msg!''}</font>
<div class="menuButton">
<a href="list" class="list">楼栋系数列表</a>
</div>
<hr/>
<div class="list">
<table width="50%">
<thead>
<tr>
<th align="center" width="150px;">
栏位
</th>
<th align="center">
值
</th>
</tr>
</thead>
<form id="addForm" action="doAdd" method="post" >
<tr>
<td><b>城市</b></td>
<td><input name="city_name" value="${city_name!}" /></td>
</tr>
<tr>
<td><b>区域</b></td>
<td><input id="region" name="region" value="${region!}" /></td>
</tr>
<tr>
<td><b>小区名称</b></td>
<td><input name="district" type="text" value="${district!}" /></td>
</tr>
<tr>
<td>楼栋名称</td>
<td><input name="building_name" value="${building_name!}" /></td>
</tr>
<tr>
<td>楼栋系数</td>
<td><input name="ratio" value="${ratio!}" /></td>
</tr>
<tr>
<td>创建人</td>
<td><input name="create_user" value="${session.realName!""}" /></td>
</tr>
<tr>
<td>备注</td>
<td><input name="note" value="${note!}" /></td>
</tr>
</form>
<tr>
<td colspan="2"><input onclick="submitForm();" type="submit" value="新增"/></td>
</tr>
</table>
</div>
</div>
<script type="text/javascript">
function submitForm(){
var city_name = $("input[name='city_name']").val();
var region = $("input[name='region']").val();
var district = $("input[name='district']").val();
var building_name = $("input[name='building_name']").val();
var ratio = $("input[name='ratio']").val();
var create_user = $("input[name='create_user']").val();
if(typeof(city_name) == "undefined" || city_name == ""){
alert("城市不能为空");
return false;
}
if(typeof(region) == "undefined" || region == ""){
alert("区域不能为空");
return false;
}
if(typeof(district) == "undefined" || district == ""){
alert("小区名称不能为空");
return false;
}
if(typeof(building_name) == "undefined" || building_name == ""){
alert("楼栋名称不能为空");
return false;
}
if(typeof(ratio) == "undefined" || ratio == ""){
alert("楼栋系数不能为空");
return false;
}
if(typeof(create_user) == "undefined" || create_user == ""){
alert("创建人不能为空");
return false;
}
$("#addForm").submit();
}
var getErrorValidate= function(mess){
var _html='<span class=\"wrong\">';
_html=_html+mess;
_html=_html+'</span>';
return _html;
}
</script>
</body>
</html>
</#compress>
@@ -0,0 +1,113 @@
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
<#compress>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<title>楼栋系数信息修改</title>
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}"/>
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
<#include "/common_js.ftl" />
<script type="text/javascript">
</script>
</head>
<body>
<#include "/layout/header.ftl" />
<div class="main-content">
<font color="red">${msg!''}</font>
<div class="menuButton">
<a href="list" class="list">楼栋系数列表</a><a href="add" class="create">新建楼栋系数</a>
</div>
<hr/>
<div class="list">
<table width="50%">
<thead>
<tr>
<th align="center" width="10%">
栏位
</th>
<th align="center" width="30%">
值
</th>
</tr>
</thead>
<form id="submitForm" action="update" method="post">
<tr>
<td>城市</td>
<td><input name="city_name" value="${buildingRatioBo.city_name!}"/></td>
</tr>
<tr>
<td>区域</td>
<td><input name="region" value="${buildingRatioBo.region!}"/></td>
</tr>
<tr>
<td>小区名称</td>
<td><input name="district" value="${buildingRatioBo.district!}"/></td>
</tr>
<tr>
<td>楼栋名称</td>
<input name="id" value="${buildingRatioBo.id!}" type="hidden"/>
<td><input name="building_name" value="${buildingRatioBo.building_name!}"/></td>
</tr>
<tr>
<td>楼栋系数</td>
<td><input name="ratio" value="${buildingRatioBo.ratio!}" /></td>
</tr>
<tr>
<td>创建人</td>
<td><input name="create_user" value="${buildingRatioBo.create_user!}" /></td>
</tr>
<tr>
<td>备注</td>
<td><input name="note" value="${buildingRatioBo.note!}" /></td>
</tr>
</form>
<tr>
<td></td>
<td><input onclick="submitForm();" type="reset" value="修改"/></td>
</tr>
</table>
</div>
</div>
<script type="text/javascript">
function submitForm() {
var city_name = $("input[name='city_name']").val();
var region = $("input[name='region']").val();
var district = $("input[name='district']").val();
var building_name = $("input[name='building_name']").val();
var ratio = $("input[name='ratio']").val();
var create_user = $("input[name='create_user']").val();
if(typeof(city_name) == "undefined" || city_name == ""){
alert("城市不能为空");
return false;
}
if(typeof(region) == "undefined" || region == ""){
alert("区域不能为空");
return false;
}
if(typeof(district) == "undefined" || district == ""){
alert("小区名称不能为空");
return false;
}
if(typeof(building_name) == "undefined" || building_name == ""){
alert("楼栋名称不能为空");
return false;
}
if(typeof(ratio) == "undefined" || ratio == ""){
alert("楼栋系数不能为空");
return false;
}
if(typeof(create_user) == "undefined" || create_user == ""){
alert("创建人不能为空");
return false;
}
$("#submitForm").attr("action", "update");
$("#submitForm").submit();
}
</script>
</body>
</html>
</#compress>
@@ -0,0 +1,120 @@
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
<#compress>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>均价管理-楼栋系数管理</title>
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
<#include "/common_js.ftl" />
</head>
<body>
<#include "/layout/header.ftl" />
<div>
<form action="../buildingRatio/list" id="_search_ratio_form">
城市:<input type="text" name="city_name" value="${city_name!}" />
区域:<input type="text" name="region" value="${region!}" />
小区:<input type="text" name="district" value="${district!}" />
<input type="submit" value="查询" />
</form>
</div>
<div class="main-content">
<font color="red">${msg!''}</font>
<div class="menuButton">
<a href="list" class="list">楼栋系数列表</a><a href="add" class="create">新建楼栋系数</a>
</div>
<hr/>
<div class="list">
<div style="padding-left:5px;">
共有楼栋系数<b>${buildingRatioBoList?size}</b>个
</div>
<table width="100%">
<thead>
<tr>
<th align="center" width="55px">
ID
</th>
<th align="center" width="100px">
城市
</th>
<th align="center" width="60px">
区域
</th>
<th align="center" width="100px">
小区名称
</th>
<th align="center" width="60px">
楼栋名称
</th>
<th align="center" width="60px">
楼栋系数
</th>
<th align="center" width="60px">
创建人
</th>
<th align="center" width="200px">
创建日期
</th>
<th align="center" width="200px">
修改日期
</th>
<th align="center" width="100px">
备注
</th>
<th align="center">
操作
</th>
</tr>
</thead>
<#if (buildingRatioBoList??&&buildingRatioBoList?size>0)>
<#list buildingRatioBoList as bo>
<tr>
<td align="center">
${bo.id!}
</td>
<td align="center">
${bo.city_name!}
</td>
<td align="center">
${bo.region!}
</td>
<td align="center">
${bo.district!}
</td>
<td align="center">
${bo.building_name!}
</td>
<td align="center">
${bo.ratio!}
</td>
<td align="center">
${bo.create_user!}
</td>
<td align="center">
<#if bo.c_date??>
${bo.c_date?string("yyyy年MM月dd日 HH:mm:ss")}
</#if>
</td>
<td align="center">
<#if bo.m_date??>
${bo.m_date?string("yyyy年MM月dd日 HH:mm:ss")}
</#if>
</td>
<td align="center">
${bo.note!}
</td>
<td align="center">
<a href="../buildingRatio/edit?id=${bo.id!}">编辑</a>
<a href="../buildingRatio/doDelete?id=${bo.id}" onclick="return confirm('确定要删除 ${bo.city_name!}-${bo.region!}-${bo.district!}-${bo.building_name!}?')">删除</a>
</td>
</tr>
</#list>
</#if>
</table>
</div>
</div>
</body>
</html>
</#compress>
@@ -90,6 +90,7 @@
<ul>
<@menuitem uri='/avgPriceUpdate/doCreatTask' text="住宅均价管理" />
<@menuitem uri='/averagerConfig/list' text="住宅系数管理" />
<@menuitem uri='/buildingRatio/list' text="楼栋系数管理" />
<@menuitem uri='/avgPriceUpdate/listQR' text="价格质量跟踪" />
<@menuitem uri='/avgPriceUpdate/listSQR' text="样本质量跟踪" />
<@menuitem uri='/comparePrice/listQR' text="调查数据跟踪" />
@@ -0,0 +1,12 @@
package com.fangjia.dal.mongo.dao.bkoff;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.bkoff.AveragerConfig;
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
/**
* @author zhaojixiang
* @since 2023/7/5
*/
public interface BuildingRatioDao extends MongoDao<BuildingRatio> {
}
@@ -0,0 +1,12 @@
package com.fangjia.dal.mongo.dao.bkoff;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.mongo.model.bkoff.AveragerConfig;
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
/**
* @author zhaojixiang
* @since 2023/7/5
*/
public class BuildingRatioDaoImpl extends MongoDaoImpl<BuildingRatio> implements BuildingRatioDao {
}
@@ -0,0 +1,113 @@
package com.fangjia.dal.mongo.model.bkoff;
import com.fangjia.dal.MongoEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.Id;
import javax.persistence.Table;
import org.springframework.format.annotation.DateTimeFormat;
import java.util.Date;
/**
* @author zhaojixiang
* @since 2023/7/5
*/
@Entity
@Table(name = "business.building_ratio")
public class BuildingRatio implements MongoEntity {
private String id;
private String city;
private String region;
private String district;
private String building_name;
private Double ratio;
@DateTimeFormat(pattern = "yyyy-MM-dd")
private Date c_date;
@DateTimeFormat(pattern = "yyyy-MM-dd")
private Date m_date;
private String create_user;
private String note;
@Id
@GeneratedValue
@Column(name = "_id")
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "district")
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
@Column(name = "building_name")
public String getBuilding_name() {
return building_name;
}
public void setBuilding_name(String building_name) {
this.building_name = building_name;
}
@Column(name = "ratio")
public Double getRatio() {
return ratio;
}
public void setRatio(Double ratio) {
this.ratio = ratio;
}
@Column(name = "c_date")
public Date getC_date() {
return c_date;
}
public void setC_date(Date c_date) {
this.c_date = c_date;
}
@Column(name = "m_date")
public Date getM_date() {
return m_date;
}
public void setM_date(Date m_date) {
this.m_date = m_date;
}
@Column(name = "create_user")
public String getCreate_user() {
return create_user;
}
public void setCreate_user(String create_user) {
this.create_user = create_user;
}
@Column(name = "note")
public String getNote() {
return note;
}
public void setNote(String note) {
this.note = note;
}
}
@@ -39,6 +39,8 @@ public class ApiUser implements SqlEntity {
private String manager;
private Integer isMonitor;
@Id
@Column(name = "ID")
public long getId() {
@@ -175,4 +177,12 @@ public class ApiUser implements SqlEntity {
public void setManager(String manager) {
this.manager = manager;
}
@Column(name = "IS_MONITOR")
public Integer getIsMonitor() {
return isMonitor;
}
public void setIsMonitor(Integer isMonitor) {
this.isMonitor = isMonitor;
}
}
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -51,7 +51,19 @@ for(def d : datas){
map.region = d['B']
def roomStr = d['E']
if(roomStr){
map.room = roomStr.toString().toDouble().toInteger()
if(roomStr.contains("-")){
def arrs = roomStr.split("-")
try{
map.room = arrs[0].toString().toInteger()
}catch(Exception e){
println "room error"
}
map.hall = arrs[1].toString().toInteger()
map.toilet = arrs[3].toString().toInteger()
}else{
map.room = roomStr.toString().toDouble().toInteger()
}
}
map.area = d['F'].toString().toDouble()
@@ -59,7 +71,13 @@ for(def d : datas){
map.avg_price = (int)(d['H'].toString().toDouble())
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
def trade_date = d['J']
map.trade_date = trade_date
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
}
map.source = "深房家族"
map.create_date = new Date()
map.m_date = new Date()
@@ -77,9 +77,8 @@ class BigdataCommunityBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成->导Oracle"
// delPgData()
// saveToPgSql(rows)
delPgData()
saveToPgSql(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
@@ -33,7 +33,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
@@ -61,17 +61,19 @@ class BigdataDealBatch extends BaseBatch{
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
// def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
def allCitys = ["福州"] //CommonDatautils.findAllCity()
for(def city : allCitys){
// if(!ccbCitys.contains(city)){
addDealPriceMatch(city);
addDealPriceMatch(city);
// }
}
// def rows = []
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -649,16 +651,21 @@ class BigdataDealBatch extends BaseBatch{
def final AtomicInteger count_ = new AtomicInteger(0)
for(def i=0;i<trades.size();i++){
def t = trades[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
final def t = trades[i]
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
count_.addAndGet(1)
try{
if(!t.match_id){
def region = RegionData.formatRegion(t.city, t.region)
def rMap = null
if(region) {
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
if(!rMap && t.address){
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
@@ -668,7 +675,12 @@ class BigdataDealBatch extends BaseBatch{
}
if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else {
if (rMap) {
println "区域不同"
}
}
}
println count_.get()+" "+t+" "+ rMap
}
@@ -678,9 +690,9 @@ class BigdataDealBatch extends BaseBatch{
latch.countDown()
}
}
}
);
// }
// }
// );
}
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
@@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
@@ -428,7 +428,7 @@ class CcbCommonUtils {
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
@@ -617,14 +617,14 @@ class CcbCommonUtils {
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', startDate))
q.put("match_id", new BasicDBObject('$exists', false))
// q.put("district_name","国贸天琴湾")
// q.put("district_name","中海寰宇天下")
def dbcursor = DEAL_PRICE_COLL.find(q)
def total = dbcursor.count()
def rets = new HashSet()
def lists = []
println city +" "+total
for(def d : dbcursor){
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id]
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id, "house_use":d.house_use,"lift_household":d.lift_household,"lift":d.lift]
}
return lists
}
@@ -28,11 +28,12 @@ class DealPriceDataProcess {
def task = new DealPriceDataProcess()
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
// task.deleteThirdPartyData()//删除第三方
task.deleteThirdPartyData()//删除第三方
// task.delDealByPe()
// task.delDealData()
task.exportThirdParty()
// task.exportThirdParty()
// task.statPpe()
}
def delDealByPe(){
@@ -379,7 +380,7 @@ class DealPriceDataProcess {
def results = []
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
for(def city : citys) {
def rets = findPeGteThirdParty3(city, 202305)
def rets = findPeGteThirdParty3(city, 202306)
for(def map : rets){
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
}
@@ -391,12 +392,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202304
def month = 202305
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202305)
def rets = findPeGteThirdParty2(city, 202306)
def total = 0
println city +" "+rets.size()
if(rets){
@@ -490,5 +491,43 @@ class DealPriceDataProcess {
return count
}
def statPpe(){
def path = "C:\\Users\\gaojuan\\Desktop\\估值数据统计.xlsx"
ExcelUtils.multiThreadingModify(path, { row ->
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
def deal_pes = []
def third_pes = []
for(def d : dates){
def map = findPeGteCity(row['B'], d)
if(map.deal_pe){
deal_pes << map.deal_pe
}
if(map.third_pe){
third_pes << map.third_pe
}
}
if(deal_pes){
row['F'] = deal_pes.sum()/deal_pes.size()
}
if(third_pes){
row['G'] = third_pes.sum()/third_pes.size()
}
println row['B'] + " "+row['F']+" "+row['G']
})
}
def findPeGteCity(cityName, yearmonth){
def sql = "SELECT gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}"
def map = [:]
pgsqlInstance.eachRow(sql,{r->
map.deal_pe = r.gujiamdltst_ppe15_pct
map.third_pe = r.gujiatrdpttstppe15pct
});
return map
}
}
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-27")
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -17,4 +17,14 @@ class PgsqlCommUtils {
});
return set
}
public static def findAllDistrictCount(def cityid){
def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
}
@@ -33,8 +33,8 @@ import java.util.regex.*;
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
@@ -45,8 +45,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
//def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
// "postgres", "postgres", "org.postgresql.Driver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
//def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
@@ -1287,50 +1287,50 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
//
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
//// def region = RegionData.formatRegion(d['A'], d['B'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
//// if(dist){
//// d['E'] = dist.region
//// d['F'] = dist._id.toString()
//// }
//// }
// if(!d['F']){
// def dist = CommonDatautils.findDistrictById(d['A'])
////
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
// if(dist){
//// d['H'] = dist.region
//// d['G'] = dist.name
// d['F'] = dist.status
// d['G'] = dist.region
// d['H'] = dist._id.toString()
// }
// }
//// if(!d['F']){
//// def dist = CommonDatautils.findDistrictById(d['A'])
//////
//// if(dist){
////// d['H'] = dist.region
////// d['G'] = dist.name
//// d['F'] = dist.status
//// }
//// }
//
// }
//},0)
def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
if(count && count==1){
def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
d['E'] = dist.city
}
// def dist = CommonDatautils.findDistrictById(d['A'])
// if(dist){
// d['B'] = dist.region
// d['C'] = dist.name
// d['J'] = dist.status
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
// if(count && count==1){
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
// d['E'] = dist.city
// }
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
// d['F'] = newRegion
},0)
//
//// def dist = CommonDatautils.findDistrictById(d['A'])
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['J'] = dist.status
//// }
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
//// d['F'] = newRegion
//},0)
//ExcelUtils.modify(fileName, {d->
// if(d['A']){
@@ -1364,6 +1364,53 @@ def findGuidedPrice(sqlInstance, id){
return count
}
def findPgDealPrice(c, pgsqlInstance, startDate){
def sql = "select count(a.gujia_comm_id) from (SELECT distinct gujia_comm_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${c}' and txn_dt>='${startDate}') a"
def count = 0
pgsqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
citys = CcbCommonUtils.citys_all
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
def rows = []
def headers = []
headers << "城市"
headers.addAll(dates)
rows << headers
for(def c : citys){
def row = []
row << c
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
// for(def d : dates){
def startDate = "2022-07-01"
def count = findPgDealPrice(c, pgsqlInstance, startDate)
row << count*100/disCount
// def map = findSampleRate(pgsqlInstance, c, d)
// row << map.third
// }
rows << row
}
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
def findSampleRate(pgsqlInstance, cityName, yearmonth){
def sql = "SELECT urbn_nm,gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}"
def map = [:]
pgsqlInstance.eachRow(sql,{r->
map.deal = r.gujiamdltst_ppe15_pct
map.third = r.gujiatrdpttstppe15pct
});
return map
}
@@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
}
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
saveToPgSql(rows)
@@ -45,6 +45,7 @@ class TradeStatLog {
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
// task.deleteDistrict()
// task.deleteDealData()
// task.deleteDealData2()
@@ -92,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-02-01"
def date = "2023-03-01"
def endDate = "2023-04-01"
def newDate = "2023-05-01"
def startDate = "2023-03-01"
def date = "2023-04-01"
def endDate = "2023-05-01"
def newDate = "2023-06-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -19,9 +19,9 @@ class CcbDataStat {
static main(args) {
def task = new CcbDataStat()
// task.run()
task.run(args)
// task.tradeRun()
task.regionDistStat()
// task.regionDistStat()
}
def regionDistStat(){
@@ -124,22 +124,37 @@ class CcbDataStat {
return count
}
def run(){
def run(args){
def citys = CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
if(d['A']) {
def citycode = cityCodeDataMap.get(d['A'])
def total = findTotalCount(citycode)
def f_count = findDistFinishDate(citycode)
def b_count = findBuildTypeCount(citycode)
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
d['C'] = f_pre
d['D'] = b_pre
d['E'] = total
println d['A'] +" "+ total +" "+f_count+" "+b_count
}
})
def rets = []
rets << ["城市","经纬度","物业类型","建筑年代","建筑类型","绿化率","容积率"]
for(def c : citys){
def ret = []
ret << c
def citycode = cityCodeDataMap.get(c)
def total = findTotalCount(citycode)
def lat_count = findLocationCount(citycode)
def f_count = findDistFinishDate(citycode)
def b_count = findBuildTypeCount(citycode)
def g_count = findVirescencePercent(citycode)
def c_count = findCubagePercentPercent(citycode)
def location_pre = DataHandleUtils.getDoubleRound(2, lat_count*100/total)
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
def g_pre = DataHandleUtils.getDoubleRound(2, g_count*100/total)
def c_pre = DataHandleUtils.getDoubleRound(2, c_count*100/total)
ret << location_pre
ret << 100
ret << b_pre
ret << f_pre
ret << g_pre
ret << c_pre
ret << total
rets << ret
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\建行供数重要字段覆盖率.xlsx", rets, "统计")
}
def findTotalCount(citycode){
@@ -168,4 +183,32 @@ class CcbDataStat {
})
return count
}
def findLocationCount(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and longitude>0";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findVirescencePercent(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and GREENINGRATE is not null";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
def findCubagePercentPercent(citycode){
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and PLOTRATIO is not null";//where cityId in (350200,150100)
def count = 0
sqlInstance.eachRow(sql, { r ->
count = r.count
})
return count
}
}
@@ -61,9 +61,9 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
cityRun(citys, startDate, endDate, cityCodeMap)
regionRun(citys, startDate, endDate, cityCodeMap)
// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
// cityRun(citys, startDate, endDate, cityCodeMap)
// regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
def cityRun(citys, startDate, endDate, cityCodeDataMap){
@@ -1184,15 +1184,15 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
}
@@ -1273,9 +1273,9 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
@@ -40,19 +40,26 @@ class HailanTest {
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new HailanTest();
task.run();
}
def run(){
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
match(inputfile)
def inputfile = "D:\\数据\\微元素\\排查表-价格对比-新.xlsx"
// addThirdPrice(inputfile)
// match(inputfile)
// tradeAvgPrice(inputfile)
// caluateTotalPrice(inputfile)
// addPrice(inputfile)
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
// addHouse(inputfile)
// stat(inputfile)
// hailanStat(inputfile)
// addFzFjwPrice(inputfile)
addFjwTobPrice(inputfile)
}
def addHouse(inputfile){
@@ -702,4 +709,72 @@ class HailanTest {
}
return prices;
}
def tradeAvgPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){
def dist = CommonDatautils.findDistrictById(row['I'])
if(dist){
def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01");
row['J'] = (int)tradeAvgPrice
}
}
})
}
def addThirdPrice(path){
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
ExcelUtils.multiThreadingModify(path, { row ->
def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate)
def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G'])
if(priceMap){
if(priceMap.containsKey("lianjia_esf_price")){
row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95)
}
}
})
}
def caluateTotalPrice(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(row['J']){
row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble())
}else if(row['K']){
row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble())
}
})
}
def addFjwTobPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['F']){
def dbo = CommonDatautils.queryDistrictPrice("福州", d['F'], d['G'], DateUtils.parseFromTightByPattern("2023-07-02","yyyy-MM-dd"))
if(dbo){
d['Q'] = dbo.avg_price_2b
}
}
})
}
def addFzFjwPrice(inputfile){
ExcelUtils.modify(inputfile, { row ->
if(row['B']) {
def q = new BasicDBObject()
q.put("city", "福州")
q.put("region", row['A'])
q.put("name", row['B'])
q.put("month", 202304)
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
if (obj) {
row['D'] = obj.avg_price
}
}
})
}
}
@@ -1,7 +1,9 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
@@ -250,7 +252,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "³¤Ì²ÃÀÊûÈýÆÚ")
// q.put("name", "鲁能大厦")
DBObject keys = new BasicDBObject();
if(fields){
@@ -737,8 +739,9 @@ class CommonDatautils {
public static def findAllDistrict(def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
// q.put("name", "中兴智汇大厦")
DBObject keys = new BasicDBObject();
if(fields){
@@ -746,7 +749,8 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
@@ -1022,7 +1026,7 @@ class CommonDatautils {
q.put("cat", cat);
q.put("region", region)
q.put("name", name);
q.put("status", 0)
q.put("visible", 0)
DBObject keys = new BasicDBObject();
if(fields){
@@ -3801,5 +3805,116 @@ class CommonDatautils {
}
static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("lianjia_esf_price", db.beike_esf_price)
}
}
return map
}
/**
* 根据经纬度获取区域
* @param city
* @param location
* @return
*/
static def getRegionByLocation(city, def location){
def json = GaoDeApiUtils.geocode(location);
def region = null;
if (json && json.info=="OK") {
def regeocode = json.regeocode
if(regeocode && regeocode.addressComponent){
region = regeocode.addressComponent.district
if(!region){
region = regeocode.addressComponent.township
}
}
}
if(region){
region = RegionData.formatRegion(city, region)
}
return region;
}
def static getAllFiles(def path) {
File file = new File(path);
File[] tempList = file.listFiles();
def files = [];
for (int i = 0; i < tempList.length; i++) {
if (tempList[i].isFile()) {
files << tempList[i];
}
if (tempList[i].isDirectory()) {
System.out.println("文件夹:" + tempList[i]);
}
}
return files;
}
}
@@ -250,8 +250,23 @@ class DataHandleUtils {
return sum;
}
public static isStandardRoad(def s){
def pattern = Pattern.compile("[路|街道|大道|巷|胡同|街|坊]\\d+(-\\d+)?(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
static main(args) {
def a = isStandardRoad("杨桥西路236号")
println a
}
}
@@ -0,0 +1,59 @@
package com.fangjia.script.district
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoUtils
import com.mongodb.BasicDBObject
//删除别名是另一个小区的主名
class MasterNameInAlias {
static main(args) {
def task = new MasterNameInAlias()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity().reverse()
def allList = []
for(def city : citys){
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
def cityList = []
cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"]
for(def dist1 : disticts){
def distList = []
for(def dist2 : disticts){
if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){
if(dist2.alias && dist2.alias.contains(dist1.name)){
def distance = ''
if(dist1.lng && dist2.lng){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d);
}
distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance]
dist2.alias.remove(dist1.name)
def d = new BasicDBObject()
d.put("_id", dist2._id)
def udata = new BasicDBObject()
udata.put("alias", dist2.alias)
udata.put("m_date", new Date())
DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
}
}
}
if(distList){
cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', '']
cityList.addAll(distList)
// cityList << []
}
}
if(cityList){
allList.addAll(cityList)
println city +" "+cityList.size()
}
}
// ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据")
}
}
@@ -0,0 +1,616 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.sql.ResultSetMetaData
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AmapAddressUpdate {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static main(args) {
def task = new AmapAddressUpdate()
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
def addr = task.formatAddress("福州", "北三环路201号")
println addr
// task.run(path)
// task.similarMatch(path)
// task.addressRun(path)
// task.addressSource(path)
// task.amapRun()
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
// task.updateData(path)
// task.updateLocation(path)//更新经纬度
}
def run(path){
def citys = ["福州"]//CommonDatautils.findAllCity()
def allrets = []
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
for (def city : citys) {
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
def rets = findAmapDetail(disticts)
allrets.addAll(rets)
}
ExcelUtils.write(path, allrets, "数据")
}
def amapRun(){
def city = "福州市"
def datas = findAllAmapByCity(city)
// datas = datas[0..10]
println "Es总量:"+datas.size()
def rets = match(datas)
ExcelUtils.write("d:\\数据\\小区\\福州高德数据匹配.xlsx", rets, "数据")
}
def match(datas){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
rets << ["ID","高德城市","高德区域","高德名称","高德地址","匹配城市","匹配区域","匹配名称","匹配类型","匹配ID"]
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = []
def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
ret << d.id
ret << d.city
ret << d.region
ret << d.name
ret << d.address
if(jsonObject != null && jsonObject.precise!=-1) {
ret << jsonObject.get("city")
ret << jsonObject.get("district")
ret << jsonObject.get("name")
ret << jsonObject.get("propertyType")
ret << jsonObject.get("id")
}
println totalCount.get() +' '+ret
rets << ret
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def findAllAmapByCity(city){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT id,cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and type like '%住宅%'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def list = []
while (rs.next()) {
def map = [:]
map.id = rs.getString("id")
map.city = city
map.name = rs.getString("name")
map.address = rs.getString("address")
map.region = rs.getString("adname")
// map.type = rs.getString("type")
// def location = rs.getObject("location")
// if(location){
// map.lng = location.x
// map.lat = location.y
// }
list << map
}
return list
}
def findAmapDetail(districts){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def row = []
row << d.city
row << d.region
row << d.name
row << d.address
if(d.addr_alias){
row << d.addr_alias.join(" ")
}else{
row << ''
}
def dbo = getAmapLocation(d.city+"市", d.region, d.name)
if(!dbo){
def newName = d.name
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
}
dbo = getAmapLocation(d.city+"市", d.region, newName)
}
// if(!dbo && d.lat){
// dbo = getAmapLocation(d.city+"市", d.region, newName, d.lat/10000000000.0, d.lng/10000000000.0)
// }
if(dbo){
row << dbo.region
row << dbo.name
row << dbo.address
row << formatAddress(d.city, dbo.address)
row << dbo.lng
row << dbo.lat
row << dbo.match_type
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
row << ""
row << ""
}
println totalCount.get() +' '+row
rets << row
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def formatAddress(def city, def address){
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
def address_str = null
if(jsonObj && jsonObj.result) {
def street = ''
def street_number = ''
if (jsonObj.result.street) {
street = jsonObj.result.street
}
if (jsonObj.result.streetNumber) {
street_number = jsonObj.result.streetNumber
}
if (street || street_number) {
address_str = street
if(street_number){
address_str = address_str + street_number
}
}
}
if(address_str && address_str.contains("交叉口")){
address_str = address_str.substring(0, address_str.indexOf("交叉口")+3)
}
return address_str;
}
def getAmapLocation(city, region, name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and type like '%住宅%'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def map = [:]
while (rs.next()) {
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
def mapName = rs.getString("name").replace("·","")
if(mapName == name && rs.getString("adname").contains(region)){
map.name = rs.getString("name")
map.address = rs.getString("address")
map.formatAddress = AddressUtil.formatAddress(city, map.address)
map.region = rs.getString("adname")
map.type = rs.getString("type")
map.match_type = "名称相同"
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
}
}
// String sql = "select * from amap_poi_detail where cityname='${city}' and name='${name}' and type like '%住宅%'"
// def loc = null
// ElasticUtils.executeSqlQuery(sql,{ data->
// println data.adname+" "+data.name +" "+data.address +" "+data.type
// if(name == data.name && data.adname.contains(region) && data.type.contains("type")){
// data.formatAddress = AddressUtil.formatAddress(city, data.address)
// loc = data
// }
// })
return map;
}
def addressRun(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
// if(row['H'] && row['H'].contains("口步行") && row['H'].contains("(")){
// row['I'] = row['H'].substring(0, row['H'].indexOf("("))
// }
if(row['J']){
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
def distance = null
if(dist1){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, row['J'].toString().toDouble(), row['K'].toString().toDouble());
}
row['M'] = distance
}
def gaodeRoadFlag = false
if(row['I']){
gaodeRoadFlag = DataHandleUtils.isStandardRoad(row['I'])
}
def dbRoadFlag = false
if(row['D']){
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
}
if(!row['N'] && row['D'] && row['I']){
if(row['D']==row['I']){
row['N'] = 100
}else if(row['E'] && gaodeRoadFlag && row['E'].contains(row['I'])){
row['N'] = 1
}
}
if(!row['N'] && !dbRoadFlag && gaodeRoadFlag){
row['N'] = 2
}
if(!row['N'] && dbRoadFlag && gaodeRoadFlag && row['I'].contains(row['D'])){
row['N'] = 3
}
if(!row['N'] && dbRoadFlag && gaodeRoadFlag){
row['N'] = 4
}
})
}
def addressSource(inputfile) {
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(!row['N'] && row['D'] && !row['O']){
def addresses = findAllEsData(row['A'], row['B'], row['C'])
if(addresses){
def dbRoadFlag = false
if(row['D']){
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
}
if(dbRoadFlag){
def list = []
list << row['D']
if(row['E']){
def arr = row['E'].split(" ")
list.addAll(arr as List)
}
for(def addr : list){
if(addresses.contains(addr)){
row['O'] = "诸葛找房"
break
}
}
}
}
}
})
}
def findAllEsData(city,region, name){
def rets = []
def sql = "select * from community where city='${city}' and name='${name}' and source='诸葛找房APP'"
rets = findDatas(region, sql)
if(!rets){
rets = findAliasDatas(city,region, name)
}
return rets;
}
def findDatas(region, sql){
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
if(d.region && d.region.contains(region) && d.address){
rets << d.address
}
})
return rets;
}
def findAliasDatas(city, region, name){
def must = []
def term1 = ["match":["city":city]]
must << term1
def should = []
should << ["match":["alias":name]]
def params = ["query":["bool":["must":must, "should":should]]]
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/community/_doc/_search");
def rets = [];
if(s) {
def doc = new groovy.json.JsonSlurper().parseText(s)
def hits = doc.hits
if(hits){
hits = hits.hits
for(def h : hits){
def d = h._source
if(d.region && d.region.contains(region) && d.address && d.source=="诸葛找房APP"){
rets << d.address
}
}
}
}
return rets;
}
def addrAndLocationDistance(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { row ->
if(!row['N'] && row['D']){
println row['D']
def dbRoadFlag = false
if(row['D']){
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
}
if(dbRoadFlag){
def gloJson = GaoDeApiUtils.geo(row['A'], row['D'])
if(gloJson) {
def glocation = null
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def map = geocodes.get(0)
if(map.formatted_address.contains(row['D'])){
def arrs = map.location.split(",")
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
def distance = null
if(dist1){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, arrs[0].toString().toDouble(), arrs[1].toString().toDouble());
}
row['P'] = distance
}
}
}
}
}
})
}
def updateData(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['Q'] && row['Q'].contains("修改地址")){
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
def udata = new BasicDBObject()
if(row['Q'].contains("可替换原地址")){
udata.put("address", row['I'])
}else if(row['Q'].contains("可新增地址别名")){
udata.put("address", row['I'])
def addrAlias = dist1.addr_alias
if(!addrAlias){
addrAlias = []
}
addrAlias << dist1.address
udata.put("addr_alias", addrAlias)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata))
}
})
}
def updateLocation(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(row['R'] && row['R']=="是"){
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
if(dist1){
def udata = new BasicDBObject()
def lat = null
def lng = null
if(row['J']){
lng = row['J'].toString().toDouble()
lat = row['K'].toString().toDouble()
}else{
def gloJson = GaoDeApiUtils.geo(row['A'], row['D'])
if(gloJson) {
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def map = geocodes.get(0)
if (map.formatted_address.contains(row['D'])) {
def arrs = map.location.split(",")
lng = arrs[0].toString().toDouble()
lat = arrs[1].toString().toDouble()
}
}
}
}
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
def lng2 = baidu.x
def lat2 = baidu.y
Long lng_new = lng *10000000000L
Long lat_new = lat *10000000000L
udata.put("lng", lng_new)
udata.put("lat", lat_new)
if(lng2){
Long lng2_new = lng2 *10000000000L
Long lat2_new = lat2 *10000000000L
udata.put("lng2", lng2_new)
udata.put("lat2", lat2_new)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata))
}
}
})
}
def similarMatch(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['F']){
def d = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
if(d && d.lat){
def dbo = getSimilarAmapLocation(d.city+"市", d.region, d.name, d.address, d.lat/10000000000.0, d.lng/10000000000.0)
if(dbo){
row['F'] = dbo.region
row['G'] = dbo.name
row['H'] = dbo.address
row['I'] = formatAddress(d.city, dbo.address)
row['J'] = dbo.lng
row['K'] = dbo.lat
row['L'] = dbo.score
row['M'] = dbo.distance
row['S'] = dbo.type
}
}
}
})
}
def getSimilarAmapLocation(city, region, name, address, lat, lng){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and MATCH(name,'${name}')")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def maxSocreObj = null
while (rs.next()) {
def map = [:]
def mapName = rs.getString("name").replace("·","")
println name +" "+ mapName
if(rs.getString("adname").contains(region)){
map.name = rs.getString("name")
map.address = rs.getString("address")
map.region = rs.getString("adname")
map.type = rs.getString("type")
map.match_type = "名称相似"
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
double nameScore = 0
double addScore = 0
double loctionScore = 0
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(lng, lat, map.lng.toString().toDouble(), map.lat.toString().toDouble());
map.distance = distance
if(map.name && name){
nameScore = CosineSimilarAlgorithm.getSimilarity(map.name, name);
}
if(map.address && address){
addScore = CosineSimilarAlgorithm.getSimilarity(map.address, address);
}
if(hasColumn(rs, "shape")){
def shape = rs.getObject("shape")
if(shape){
def coordinates = shape.getObject("coordinates")
// def pointList = InsidePolygon.createPoint2(points);
def flag = InsidePolygon.isInArea(lng, lat, coordinates)
if(flag){
loctionScore = 0.5
}
}
}else{
if(distance < 500 && (nameScore>0.5 || addScore>0.5)){
loctionScore = 0.3
}
}
def score = nameScore*0.3 + addScore*0.2 + loctionScore
map.score = score
if(!maxSocreObj){
if(score>0){
map.formatAddress = AddressUtil.formatAddress(city, map.address)
maxSocreObj = map
}
}else{
println score +" "+maxSocreObj.score
if(maxSocreObj && score>maxSocreObj.score){
map.formatAddress = AddressUtil.formatAddress(city, map.address)
maxSocreObj = map
}
}
}
}
return maxSocreObj;
}
def hasColumn(ResultSet rs, String columnName) {
ResultSetMetaData rsmd = rs.getMetaData();
int columns = rsmd.getColumnCount();
for (int x = 1; x <= columns; x++) {
if (columnName.equals(rsmd.getColumnName(x))) {
return true;
}
}
return false;
}
}
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.lang.StringUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -321,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{
def s = str.toString();
index++;
if(s && s!="null"){
s = s.replace('"','\\"')
// if(s.toString().contains(",")){
// s = csvHandlerStr(s.toString())
//
@@ -60,7 +60,8 @@ class DistrictSellStatBatch extends BaseBatch {
}
def calcMonthlyPrice(){
def citys = CommonUtils.findAllCity()
//Ö»Óи£ÖÝʹÓÃtob¼Û¸ñ
def citys = CommonUtils.findAllCity()
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
@@ -142,17 +143,15 @@ class DistrictSellStatBatch extends BaseBatch {
if(weekendPriceObj.value_type!=1){
if(dist.city == "¸ŁÖÝ"){
if(Math.abs(mom)*100>3){
if(dist.region != "Âíβ"){
def randMarkup = nextDouble(0, 3)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
}else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
}
// if(Math.abs(mom)*100>3){
// def randMarkup = nextDouble(0, 3)
// if(mom < 0){
// mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
// }else{
// mom = DataUtil.roundDouble(randMarkup/100, 4)
// }
// curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
// }
}else{
if(Math.abs(mom)*100>10){
def randMarkup = nextDouble(0, 10)
@@ -232,6 +231,11 @@ class DistrictSellStatBatch extends BaseBatch {
def dbcursor = DISTRICT_STAT.find(q)
def priceMap = [:]
for(def d : dbcursor){
if(city == "¸£ÖÝ"){
if(d.avg_price_2b){
d.avg_price = d.avg_price_2b
}
}
priceMap.put(d.region +"##"+d.district_name, d)
}
return priceMap
@@ -25,7 +25,7 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
// task.regionPriceTrendRun()
task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
}
@@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{
// sb.append(s+ "@|@")
// }
// }
sb.append(StringUtils.join(row.toArray(), "@|@"))
def str = StringUtils.join(row.toArray(), "@|@")
str = str.replace('"','\\"')
sb.append(str)
sb.append(System.getProperty("line.separator"));
println index
@@ -2,6 +2,9 @@ package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.JsonUtils
import com.fangjia.core.util.Message
import com.fangjia.core.util.MessageInfo
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsUtils
import com.fangjia.script.util.DateUtils
@@ -9,26 +12,71 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.io.FileUtils
import org.bson.types.ObjectId
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
import com.fangjia.core.util.HttpRequest
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
def city = RegionData.formatCity("余姚")
d['D'] = city
if(city){
d['E'] = RegionData.formatRegion(city, d['C'])
}
// if(!d['D']){
// def citys = CommonDatautils.getCityByRegion(d['A'])
// if(citys && citys.size()==1){
// d['F'] = citys.get(0)
// }
//Message message = new Message();
//message.setMsgtype("text");
//message.setText(new MessageInfo("测试"));
//URL url = new URL("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430");
//// 建立http连接
//HttpURLConnection conn = (HttpURLConnection) url.openConnection();
//conn.setDoOutput(true);
//conn.setDoInput(true);
//conn.setUseCaches(false);
//conn.setRequestMethod("POST");
//conn.setRequestProperty("Charset", "UTF-8");
//conn.setRequestProperty("Content-Type", "application/Json; charset=UTF-8");
//conn.connect();
//OutputStream out = conn.getOutputStream();
//String textMessage = JsonUtils.toJson(message);
//byte[] data = textMessage.getBytes("UTF-8");
//out.write(data);
//out.flush();
//out.close();
//System.out.println(conn.getResponseCode());
//InputStream ins = conn.getInputStream();
//byte[] data1 = new byte[ins.available()];
//ins.read(data1);
//System.out.println(new String(data1));
//HttpRequest request = new HttpRequest()
//def params = [:]
//params.put("","")
//request.post("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430","", "utf-8")
//sendMarkdownMessage("租金", "测试")
//static void sendMarkdownMessage(String title, String msg){
// HttpRequest request = new HttpRequest()
// JSONObject requestBody = new JSONObject()
// JSONObject markdown = new JSONObject()
// markdown.put("title", title)
// markdown.put("text", msg)
// requestBody.put("markdown", markdown)
// requestBody.put("msgtype", "markdown")
// request.postWithBody("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430", requestBody, null, "UTF-8")
//}
//ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
// def city = RegionData.formatCity("余姚")
// d['D'] = city
// if(city){
// d['E'] = RegionData.formatRegion(city, d['C'])
// }
})
//
//// if(!d['D']){
//// def citys = CommonDatautils.getCityByRegion(d['A'])
//// if(citys && citys.size()==1){
//// d['F'] = citys.get(0)
//// }
//// }
//})
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
//def m = [:]
@@ -188,6 +236,24 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
//},0)
def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
if(files){
for(def filepath : files){
def rets = read(filepath);
}
}
def read(path){
File file = new File(path);
def rets = []
file.eachLine{it->
rets << it
}
return rets;
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -29,10 +29,11 @@ class AddAvgpriceBatch {
}
def test(){
def districts = CommonDatautils.findDistrictByCityAndRegions("¸£ÖÝ","district", ["Âíβ"], ["city","region","name", "estate_type2"])
def districts = CommonDatautils.findAllDistricts("¸£ÖÝ","district", ["city","region","name", "estate_type2"])
// def districts = CommonDatautils.findDistrictByCityAndRegions("¸£ÖÝ","district", ["Âíβ"], ["city","region","name", "estate_type2"])
for(def dist : districts){
//发重算均价命令
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd'))
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-06-30','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-07-05','yyyy-MM-dd'))
}
}
def run(){
@@ -8,6 +8,7 @@ import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Matcher
import java.util.regex.Pattern
@@ -20,8 +21,10 @@ class FiledStrand {
def updateIndex = 0;
static main(args) {
def task = new FiledStrand();
task.run();
// def task = new FiledStrand();
// task.run();
// task.updateOpenDate();
}
@@ -265,13 +268,58 @@ class FiledStrand {
static def processArea(all_area){
if(all_area){
def v = getValue(all_area)
if(v>=1000 && v<=10000000){
if(v>=5000 && v<=10000000){
return v.toString().toDouble().toInteger()
}
}
return null
}
static def processStandardHeight(height){
if(height){
def v = getValue(height)
if(v>=2.4 && v<=10){
return v.toString()
}
}
return null
}
static def processElevators(elevators){
if(elevators){
def v = getValue(elevators)
if(v>0 && v<=100){
return v.toString()
}
}
return null
}
static def processStandardArea(area){
if(area){
area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","")
area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll("");
return area
}
return null
}
static def processCarSiteRent(rent){
if(rent){
def v = getNumberRange(rent)
if(v){
if(v=="0"){
return null
}else{
return v.toString()
}
}
return null
}
return null
}
static def processName(name){
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
if (name.contains('(') && name.contains(')')){
@@ -287,25 +335,25 @@ class FiledStrand {
if (name.find('【')!=-1 && name.find('】')!=-1){
def rep_str = name[name.index('【'):name.rfind('】')+1]
def rep_str = name.subsring(name.index('【'),name.rfind('】')+1)
name = name.replace(rep_str, '')
}
if (name.find('【')!=-1 && name.find('】')!=-1){
def rep_str = name[name.index('【'):name.rfind('】')+1]
def rep_str = name.subsring(name.index('【'),name.rfind('】')+1)
name = name.replace(rep_str, '')
}
if (name.find('(')!=-1 && name.find(')')!=-1){
def rep_str = name[name.index('('):name.rfind(')')+1]
def rep_str = name.subsring(name.index('('),name.rfind(')')+1)
name = name.replace(rep_str, '')
}
if (name.find('「')!=-1 && name.find('」')!=-1){
def rep_str = name[name.index('「'):name.rfind('」')+1]
def rep_str = name.subsring(name.index('「'),name.rfind('」')+1)
name = name.replace(rep_str, '')
}
return name
@@ -338,7 +386,7 @@ class FiledStrand {
if(v){
if([40,50,70].contains(v)){
try {
return v.toString()+"年";
return v.toString();
}catch(Exception e){
println e.getMessage()
}
@@ -378,6 +426,16 @@ class FiledStrand {
return null
}
static def processStandardAddress(def esAddress, def city){
if(esAddress){
def formatAddress = AddressUtil.formatAddress(city, esAddress)
if(formatAddress && isStandardRoad(formatAddress)){
return formatAddress
}
}
return null
}
static def processBlock(city, region, def block){
if(block){
if(block.contains("无") || block.toString().length()>10){
@@ -464,6 +522,14 @@ class FiledStrand {
return false
}
static def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
static def addBuildAreaToData(def dist, def updateFile){
@@ -629,7 +695,7 @@ class FiledStrand {
println "estateCharege 转不了 "+estateCharege
}
}
return estateCharege;
return null;
}
static def processHouseholdCount(def household_count, def estateType2){
@@ -666,6 +732,34 @@ class FiledStrand {
return null
}
static def processFloor(floor){
def v = getValue(floor.toString());
if(v){
if(v>=1 && v<=130){
try {
return v.toString();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processRealUsePercent(percent){
def v = getValue(percent.toString());
if(v){
if(v>=20 && v<=80){
try {
return v.toString();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def processFinishBuildingDate(def finishBuildingDate){
if(finishBuildingDate){
Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}");
@@ -677,24 +771,37 @@ class FiledStrand {
Pattern pat3 = Pattern.compile("\\d{4}");
def matcher3 = pat3.matcher(finishBuildingDate);
def dateStr = null
if (matcher1.find()) {
return matcher1.group().replace("年","-").replace("月","-")
dateStr = matcher1.group().replace("年","-").replace("月","-").replace(".","")
}else if(matcher2.find()){
return matcher2.group().replace("年","-")
dateStr = matcher2.group().replace("年","-").replace(".","-")
}else if(matcher3.find()){
return matcher3.group()
dateStr = matcher3.group().replace(".","-")
}
if(dateStr){
dateStr = dateStr.substring(0, 4)
}
return dateStr
}
return null;
}
static def processCompany(val){
if(val){
if(val.length()>20){
return null
}else{
if(val.contains("无")){
if(val.contains("暂无")){
return null
}
if(val == "无"){
return null
}
if(val.isNumber()){
return null
}
def fiterChars = [",",".","?","/","数据",'-','?','&#160;','&nbsp;']
@@ -771,4 +878,31 @@ class FiledStrand {
}
return null;
}
static def patternParse(String keyword, String patternStr) {
Pattern pattern = Pattern.compile(patternStr);
Matcher matcher = pattern.matcher(keyword);
String num = "";
if (matcher.find()) {
if (StringUtils.isNotBlank(matcher.group(0))) {
num = matcher.group(1).trim();
}
}
return num;
}
static def getNumberRange(def input) {
Pattern pattern = Pattern.compile("\\d+(-\\d+)?");
Matcher matcher = pattern.matcher(input);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list.size()>0){
return list.get(0);
}
return null;
}
}
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-04-01"
static def statDate = "2023-05-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -65,7 +65,7 @@ class AddDistrictElevatorBatch extends BaseBatch{
}
def newHouses = analyElevator(haveHouses, dist.estate_type2)
if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 10层以上:"+newHouses.size()
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size()
BasicDBObject udata = new BasicDBObject()
if(dist.estate_type2 == "±ðÊû"){
udata.put("is_elevator", false)
@@ -1,9 +1,14 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.statistics.poi.AmapPoi
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.HttpRequestUtil
import com.mongodb.DBObject
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.StringUtils
class FangjiaClientUtils {
@@ -153,4 +158,18 @@ class FangjiaClientUtils {
return columnValMap;
}
public static def saveOrUpdateDistrict(data) {
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
if (json.status == 0) {
return "³É¹¦"
}else if(json.status == 1){
return json.msg
}
}
}
@@ -4,14 +4,25 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 写字楼字段清洗入库(自动入库)
@@ -19,6 +30,28 @@ import java.util.concurrent.Executors
@BatchOwner(Watcher.GaoJuan)
class OfficeFieldCleanBatch extends BaseBatch{
//"build_date":"build_year", "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address",
// "live_date":"live_date", "loop_line_info":"loop_line"
def fieldsMap = ["estate_charge":"property_fee","block":"block","estate_company":"property_company","developer":"developer",
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area"
,"finish_building_date":"finish_building_date","property_life":"property_life","standard_area":"standard_area","total_floor":"total_floor",
"standard_height":"floor_height", "real_use_percent":"room_rate", "loop_line_info":"ring_location","car_site_count":"parking",
"parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area"]//key:seaweed表字段 value:es表字段
//"安居客":"安居客app",
def sourceMap = ["房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"]
def fieldMap = ["编号":"_id","城市":"city","区域":"region","名称":"name","别名":"alias","租金":"rent_price","板块":"block","物业公司":"estate_company","物业费":"estate_charge",
"地址":"address","地址别名":"addr_alias","经度":"lng","纬度":"lat","类型":"estate_type2","建筑面积":"build_area","等级":"bed_kind",
"开发商":"developer","占地面积":"all_area","标准层面积":"standard_area","产权年限":"property_life","总层数":"total_floor","是否涉外":"is_foreign",
"容积率":"cubage_percent","绿化率":"virescence_percent","货梯数":"goods_elevator_count","竣工时间":"finish_building_date","客梯数":"person_elevators_count",
"标准层高":"standard_height","净高":"clear_height","大堂层高":"lobby_height","得房率":"real_use_percent","是否可分割":"divisible","环线位置":"loop_line_info",
"车位描述":"car_site","车位数量":"car_site_count","地上车位":"land_car_site","地下停车位":"underground_car_site","车位租金":"parking_fee","电梯数量":"elevators_count",
"是否注册":"is_register","开间面积":"bay_area","楼层信息":"floor_info"]
static DBCollection BLOCKPOLYLINECOLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border2", "mongo.db.fangjia.com")
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
@@ -27,12 +60,182 @@ class OfficeFieldCleanBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity().reverse()
for(def c : citys){
def offices = findAllOffice(c)
strandField(offices)
// def str = "1200平方/层,1067㎡,1650-1900平米"
// def citys = CommonDatautils.findAllCity().reverse()
// for(def c : citys){
// def offices = findAllOffice(c)
// strandField(offices)
// }
// addBlock()
// addFieldValue()
// updateErrorField()
export()
test()
// addOffice()
}
def export(){
def offices = CommonDatautils.findAllFiledDistrict("office", "上海")
def fields = fieldMap.keySet().asList()
def rets = []
rets << fields
for(def i=0;i<offices.size();i++){
def data = offices[i]
if(data.region == "浦东"){
def row = []
fieldMap.each{k,v->
def val = data.get(v)
if(k == "编号"){
row << val.toString()
}else if(v=="lat" || v=="lng"){
row << val.toString().toDouble()/10000000000.0
}else if(v == "divisible"){
if(val==1){
row << "是"
}else if(val==0){
row << "否"
}else{
row << ''
}
}else if(v.contains("alias")){
if(val){
row << val.join(" ")
}else{
row << ''
}
}else{
row << val
}
}
rets << row
}
}
ExcelUtils.write("D:\\数据\\写字楼\\中远.xlsx",rets, "数据")
}
def test(){
def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格租金.xlsx",0 , 1)
def map = [:]
for(def d : datas){
map.put(d['A']+"#"+d['B']+'#'+d['C'], d)
}
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\中远.xlsx",{d->
def row = map.get(d['B']+"#"+d['C']+'#'+d['D'])
d['F'] = row['F']
})
}
def updateErrorField(){
// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def offices = CommonDatautils.findAllDistricts("上海","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
for(def i=0;i<offices.size();i++){
final def o = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
updateDistField(o)
println totalCount.get() +" "+o.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def updateDistField(o){
def addr_alias = o.addr_alias
def udata = new BasicDBObject()
if(o.property_life){
def newVal = FiledStrand.processPropertyLife(o.property_life)
udata.put("property_life", newVal)
}
if(o.total_floor){
def newVal = FiledStrand.processFloor(o.total_floor)
udata.put("total_floor", newVal.toString())
}
if(o.finish_building_date){
def newVal = FiledStrand.processFinishBuildingDate(o.finish_building_date)
udata.put("finish_building_date", newVal.toString())
}
if(o.real_use_percent){
def newVal = FiledStrand.processRealUsePercent(o.real_use_percent)
udata.put("real_use_percent", newVal.toString())
}
if(o.underground_car_site){
def newVal = FiledStrand.processCarSite(o.underground_car_site)
udata.put("underground_car_site", newVal.toString())
}
if(o.land_car_site){
def newVal = FiledStrand.processCarSite(o.land_car_site)
udata.put("land_car_site", newVal.toString())
}
if(o.parking_fee){
def newVal = FiledStrand.processCarSiteRent(o.parking_fee)
udata.put("parking_fee", newVal)
}
if(addr_alias){
def addrs = []
for(def arr : addr_alias){
def addr = FiledStrand.processStandardAddress(arr, o.city)
if(addr){
addrs << addr
}
}
if(addrs!=addr_alias){
if(addrs){
udata.put("addr_alias", addrs)
}else{
udata.put("addr_alias", null)
}
}
}
// if(o.estate_charge && !o.estate_charge.isNumber()){
//// def val = o.estate_charge.toString().toDouble()
//// if(val>40 || val < 0.01){
//// println val
// udata.put("estate_charge", null)
//// }
// }
if(udata){
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',o._id),new BasicDBObject('$set',udata))
}
}
def addFieldValue(){
def citys = ["上海"]
def analysRets = []
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("office", c)
if(totalCount){
analysRets = addFieldFromMatch(c, totalCount, analysRets)
}
}
}
@@ -65,15 +268,431 @@ class OfficeFieldCleanBatch extends BaseBatch{
}
def findAllOffice(city){
def sql = "select * from office where city='${city}'"
def rets = []
ElasticUtils.executeSqlQuery(sql, { d->
rets << d
def findEsDatas(c, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from office where city='${c}' and source='${esWebSource}'", {r->
list << r
})
return list
}
def matchDist(city, region, name){
def dist = CommonDatautils.findDistrict(city, region, name, "office")
return dist;
}
def matchApi(esRets){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(esRets.size())
for(def i=0;i<esRets.size();i++){
final def data = esRets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = matching(data)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def matching(data){
def name = data.name
if(data.name && data.name.contains("·")){
name = data.name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(data.city, data.region, name)
if(!dist){
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"office")
flag = "映射别名匹配"
}
if(!dist) {
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "office")
if(rets && rets.size()==1){
flag = "别名匹配"
dist = rets[0]
}
}
if(dist){
data.flag = flag
data.match_id = dist._id.toString()
data.match_region = dist.region
data.match_name = dist.name
data.credit = 0.95
}else{
def jsonObject = OpenApiClient.addressMatch(data.city, name, "office", false,data.region, null)
if(jsonObject!=null && jsonObject.precise==-1){
if(data.address && isStandardRoad(data.address)){
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "office", false,data.region, name)
}else{
data.flag = "匹配接口-地址"
}
}else{
data.flag = "匹配接口-名称"
}
if(jsonObject != null && jsonObject.precise!=-1) {
data.match_id = jsonObject.get("id")
data.match_region = jsonObject.get("district")
data.match_name = jsonObject.get("name")
data.credit = jsonObject.get("credit")
}
}
if(data.match_id){
return data
}
return null
}
def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
def selectRestByCredit(matchRets){
def matchRets1 = []
for(def r : matchRets){
if(r.credit && r.credit>=0.9){
matchRets1 << r
}
}
return matchRets1
}
def addFieldFromMatch(c, totalCount, analysRets){
def fields = fieldsMap.keySet() as List
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource)
// esRets = esRets[0..100]
def matchRets1 = matchApi(esRets)
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
def updateRets = processFieldsFromMatch(c , matchRets2)
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+ DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
if(esRets){
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
}
def analyRet = []
analyRet << seaweedmapSource
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analyRet << matchRets1.size()
analyRet << matchRets2.size()
analyRet << pre1
analyRet << pre2
analysRets << analyRet
}
return analysRets;
}
def fieldUpdate(total,updateRets){
def countMap = [:]
for(def rm : updateRets){
def valMap = rm.value
fieldsMap.each{seaweedfield,esField->
if(valMap.containsKey(seaweedfield)) {
if(countMap.containsKey(seaweedfield)) {
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
}else{
countMap.put(seaweedfield, 1)
}
}
}
}
println countMap.toString()
def precentMap = [:]
fieldsMap.each() {key,skey->
if(countMap.containsKey(key)) {
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
}else {
precentMap.put(key, 0)
}
}
return precentMap;
}
def processFieldsFromMatch(c,matchRets){
def fields = fieldsMap.keySet() as List
fields << "city"
fields << "estate_type2"
def fieldValueMap = [:]
for(def matchDist : matchRets){
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
continue
}
def newValues = strandField(c, dist, matchDist)
if(newValues){
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
}
}
return fieldValueMap;
}
def strandField(c,dist, matchDist){
def newValues = [:]
fieldsMap.each{seaweedfield,esField->
def val = dist.get(seaweedfield)
if(!val){
def esVal = matchDist.get(esField)
if(esVal){
def newVal = null
if(seaweedfield == "estate_charge"){
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
newVal = newVal.toString()
}
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
newVal = FiledStrand.processFinishBuildingDate(esVal)
}
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
newVal = FiledStrand.processCompany(esVal)
}
if(seaweedfield =="virescence_percent"){
newVal = FiledStrand.processVirescencePercent(esVal)
}
if(seaweedfield =="cubage_percent"){
newVal = FiledStrand.processCubagePercent(esVal)
}
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
newVal = FiledStrand.processArea(esVal.toString())
if(newVal){
newVal = newVal.toString()
}
}
if(seaweedfield =="household_count"){
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
}
if(seaweedfield =="loudong_count"){
newVal = FiledStrand.processLoudongCount(esVal)
}
if(seaweedfield =="car_site_count"){
newVal = FiledStrand.processCarSite(esVal)
}
if(seaweedfield =="car_site"){
newVal = FiledStrand.processCarSiteDesc(esVal)
}
if(seaweedfield =="property_life"){
newVal = FiledStrand.processPropertyLife(esVal)
}
if(seaweedfield =="alias"){
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
}
if(seaweedfield =="address"){
newVal = FiledStrand.processAddress(esVal, dist.city)
}
if(seaweedfield =="block"){
newVal = FiledStrand.processBlock(dist.city, dist.region, esVal)
}
if(seaweedfield == "build_type"){
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
}
if(seaweedfield == "land_property"){
newVal = FiledStrand.processlandProperty(esVal)
}
if(seaweedfield == "build_structure"){
newVal = FiledStrand.processBuildStructure(esVal)
}
if(seaweedfield == "total_floor"){
newVal = FiledStrand.processFloor(esVal)
}
if(seaweedfield == "real_use_percent"){
newVal = FiledStrand.processRealUsePercent(esVal)
}
if(seaweedfield == "parking_fee"){
newVal = FiledStrand.processCarSiteRent(esVal)
}
if(seaweedfield == "elevators_count"){
newVal = FiledStrand.processElevators(esVal)
}
if(seaweedfield == "standard_area"){
newVal = FiledStrand.processStandardArea(esVal)
}
if(seaweedfield == "standard_height"){
newVal = FiledStrand.processStandardHeight(esVal)
}
if(seaweedfield == "bay_area"){
newVal = esVal
}
if(newVal){
newValues.put(seaweedfield, newVal)
}
}
}
}
return newValues;
}
def updateDB(c, updateRets){
for(def rm : updateRets) {
def id = rm.key
def valMap = rm.value
BasicDBObject udata = new BasicDBObject()
valMap.each {k,v->
udata.put(k,v)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
}
}
def addBlock(){
def rets = findAllBlockRange()
def offices = CommonDatautils.findAllDistricts("上海", "office", ["city","region","name","block","lat2", "lng2"])
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
for(def i=0;i<=offices.size();i++){
final def office = offices[i]
if(office && !office.block){
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def lastObj = null
for (def r : rets) {
def flag = InsidePolygon.isInArea(office.lng2 / 10000000000.0, office.lat2 / 10000000000.0, r.points)
if (flag && r.name) {
lastObj = r;
break;
}
}
if (lastObj) {
def udata = [:]
udata.put("block", lastObj.name)
udata.put("status", "0")
udata.put("cat", "office")
udata.put("username", "高娟")
udata.put("id", office._id.toString())
udata.put("loopLineInfo", lastObj.loop)
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println s
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
}
latch.await()
}
def findAllBlockRange(){
def rets = []
def q = new BasicDBObject()
def dbcursor = BLOCKPOLYLINECOLL.find(q)
for(def d : dbcursor) {
def pointList = InsidePolygon.createPoint3(d.border);
d.points = pointList
rets << d
}
return rets;
}
def saveOptLog(city, region, name, datas){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category","office");
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", "update")
obj.put("data", datas)
obj.put("user_name", "高娟")
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
def addOffice(){
def city = "上海"
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\bd_file_insert(待入库).xlsx",{d->
def udata = [:]
def formatAddress = AddressUtil.formatAddress(city, d['B'])
udata.put("city", city)
udata.put("region", d['A'])
udata.put("name", d['D'])
udata.put("address", formatAddress)
long lng = (d['L'].toString().toDouble()*10000000000).longValue()
long lat = (d['K'].toString().toDouble()*10000000000).longValue()
udata.put("lat", lat)
udata.put("lng", lng)
udata.put("status", "0")
udata.put("cat", "office")
udata.put("username", "高娟")
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println d['D'] + s
})
return rets
}
}
@@ -10,6 +10,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
*处理地址在地址别名中有重复
*/
class SameDistName {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
@@ -18,13 +21,13 @@ class SameDistName {
def task = new SameDistName()
// task.run()
//地址和地址别名完全一致
// task.addressPocess()
task.addressPocess()
}
def addressPocess(){
def citys = CommonDatautils.findAllCity();
def citys = ["福州"]//CommonDatautils.findAllCity();
for(def c : citys){
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"])
@@ -0,0 +1,45 @@
package com.fangjia.script.gj.house
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class RemoveHouse extends BaseBatch{
DBCollection SEABIRD_COLL = MongoUtils.getCollectionByName("fangjia", "seabird", "mongo.db.fangjia.com")
static main(args) {
runBatch(new RemoveHouse(), args)
}
@Override
void process(Object args) {
def start = new Date()
def endDate = DateUtils.parseFromTightYYYYMMDD("2023-02-01")
def q = new BasicDBObject()
q.put("c_date", new BasicDBObject('$lte', endDate))
def dbcursor = SEABIRD_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << d._id
if(rets.size()==200){
def delquery = new BasicDBObject()
delquery.put("_id", new BasicDBObject('$in', rets))
SEABIRD_COLL.remove(delquery)
rets = []
print "."
}
}
def end = new Date()
println "¿ªÊ¼Ê±¼ä£º"+start
println "½áÊøÊ±¼ä£º"+end
}
}
@@ -0,0 +1,121 @@
package com.fangjia.script.gj.location
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 判断小区是否在城市范围内
*/
class DistrictLocationBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static DBCollection RANGECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
static main(args) {
def task = new DistrictLocationBatch()
// task.run()
task.deleteDist()
}
def deleteDist(){
ExcelUtils.multiThreadingModify("d:\\数据\\小区\\非本城市小区.xlsx",{d->
DBObject q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['B'])
q.put("name", d['C'])
q.put("cat", 'district')
q.put("status", 0)
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
})
}
def run() {
def citys = CommonDatautils.findAllCity()
def cityMap = findCityRange()
def allrets = []
for (def city : citys) {
def points = cityMap.get(city)
def pointList = InsidePolygon.createPoint3(points);
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "lat", "lng"])
def rets = findNotInCityDistricts(city, disticts, pointList)
allrets.addAll(rets)
}
ExcelUtils.write("d:\\数据\\小区\\非本城市小区.xlsx", allrets, "经纬度不在此城市中")
}
def findNotInCityDistricts(city, districts, pointList){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(dist.lng){
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
if(!flag) {
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
ret << dist.address
println ret
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
println city+" "+ rets.size() +" "+districts.size()
return rets
}
def findCityRange(){
def q = new BasicDBObject()
def dbcursor = RANGECOLL.find(q)
def cityMap = [:]
for(def d : dbcursor){
cityMap.put(d.fjw_city, d.polyline)
}
return cityMap
}
def isInArea(lng, lat, pointList){
def flag = false;
for(def pointArr : pointList){
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
if(flag){
break;
}
}
return flag;
}
}
@@ -0,0 +1,84 @@
package com.fangjia.script.gj.location
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
class GaodeCityRange {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
static main(args) {
def task = new GaodeCityRange()
task.run()
}
def run(){
def citys = CommonDatautils.findAllCity()
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
HttpRequest request = new HttpRequest()
def rows = []
citys.each{c->
if(!citys1.contains(c)){
c = c.replace("州","")
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
if(resp.containsKey("districts")){
def datas = resp.get("districts")
for(def d : datas){
println c +" "+ d.name+" "+d.level
if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){
rows << d
}else{
if(d.level=="city"){
rows << d
}
}
}
}
}
}
// citys.each{c->
// println c
// JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
// if(resp.containsKey("districts")){
// def datas = resp.get("districts")
// for(def d : datas){
// println c +" "+ d.name+" "+d.level
// if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){
// rows << d
// }else{
// if(d.level=="city"){
// rows << d
// }
// }
// }
// }
// }
if(rows){
rows.each {r->
def obj = new BasicDBObject()
obj.put("level", r.level)
obj.put("center", r.center)
obj.put("center", r.center)
obj.put("polyline", r.polyline)
obj.put("name", r.name)
obj.put("adcode", r.adcode)
obj.put("citycode", r.citycode)
def city = RegionData.formatCity(r.name)
obj.put("fjw_city", city)
POLYLINECOLL.save(obj)
}
}
}
}
@@ -3,6 +3,9 @@ package com.fangjia.script.gj.location
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
@@ -25,28 +28,122 @@ import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern;
class LocationTest {
/**
* 根据区域围栏修正区域
*/
@BatchOwner(Watcher.GaoJuan)
class RegionLocationUpdateBatch extends BaseBatch{
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
// Date date=new Date(1622802609);
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// String str = format.format(date);
// System.out.println(str);
runBatch(new RegionLocationUpdateBatch(), args);
}
def task = new LocationTest()
task.run()
@Override
public void process(Object args) {
regionUpdate()
// task.run()
// task.run2()
// task.run3()
// task.run4()
// task.run5()
}
def regionUpdate(){
def citys = CommonDatautils.findAllCity()
def rets = []
def index = 0
for(def city : citys){
index ++
def regionPointsMap = findRegionRange(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
def datas = findInRegionDistricts(city, districts, regionPointsMap)
println "【${index}】"+city +" 经纬度处理完成"
rets.addAll(datas)
}
ExcelUtils.write("D:\\数据\\小区\\区域纠正.xlsx", rets, "数据")
}
def findInRegionDistricts(city, districts, regionPointsMap){
def index = 0
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def regionMap = [:]
regionPointsMap.each{k,v->
def pointList = InsidePolygon.createPoint3(v);
regionMap.put(k, pointList)
}
def keys = regionPointsMap.keySet()
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
if(dist.lng){
totalCount.addAndGet(1)
def pointList = regionMap.get(dist.region)
def flag = InsidePolygon.isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
def newRegion = null
if(!flag){
for(def m : regionPointsMap){
if(m.key != dist.region){
pointList = regionMap.get(m.key)
flag = InsidePolygon.isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
if(flag){
newRegion = m.key
break;
}
}
}
}
if(flag){
if(newRegion){
println dist.city +" "+dist.region+" "+dist.name+" 【${newRegion}】"
def udata = [:]
udata.put("region", newRegion)
udata.put("status", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
udata.put("id", dist._id.toString())
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
def opt = "update"
if(s && s!='成功'){
if(s && s.contains("该区域小区已存在")){
def delUdata = new BasicDBObject()
delUdata.put("status", 1)
delUdata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
saveOptLog(dist.city, dist.region, dist.name)
opt = "delete"
}
}
rets << [dist.city, dist.region, dist.name, newRegion, opt]
}
}else{
println dist.city +" "+dist.region+" "+dist.name+" 没有找到对应的区域"
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def run(){
def city = "º¼ÖÝ"
def regionPointsMap = findRegionRange(city)
@@ -446,12 +543,11 @@ class LocationTest {
def findRegionRange(city){
def q = new BasicDBObject()
q.put("city",city+"ÊÐ")
q.put("city",city)
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline)
regionMap.put(d.fjw_region, d.polyline)
}
return regionMap
}
@@ -525,4 +621,22 @@ class LocationTest {
return rets
}
def saveOptLog(city, region, name){
def obj = new BasicDBObject();
obj.put("city",city);
obj.put("region",region);
obj.put("district_name",name);
obj.put("category","district");
obj.put("c_date", new Date())
obj.put("m_date", new Date())
obj.put("type", "remove")
obj.put("user_name", "高娟")
obj.put("origin", "script")
OPERATION_LOG_COLL.save(obj)
}
}
@@ -1,25 +1,19 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.location
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.apache.commons.collections.CollectionUtils
import org.apache.commons.lang.math.RandomUtils
import org.apache.commons.lang3.StringUtils
import org.bson.Document
import java.util.regex.Pattern
/**
* 抓取区域边框
* 高德抓取区域边框
*/
class RegionPointCrawl {
@@ -28,13 +22,19 @@ class RegionPointCrawl {
static main(args) {
def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
//4个直辖市 北京市、上海市、天津市、重庆市
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
def provinces = ["甘肃省"]
def index = 0
for(def p : provinces){
index ++
println index +"/"+provinces.size() +" "+p
task.crawlCityAndDistrictCode(p);
println "城市 区域 code 完成"
task.run(p);
println "区域围栏 完成"
}
}
@@ -43,8 +43,8 @@ class RegionPointCrawl {
crawlCity(province)
}
def run(city){
def cityCodes = findCityCode(city);
def run(pro){
def cityCodes = findCityCode(pro);
if(cityCodes){
for(def cityMap : cityCodes){
def codeMap = findAllDistrictCode(cityMap.city_code);
@@ -59,7 +59,8 @@ class RegionPointCrawl {
if(ret.size()>0){
def standardCity = RegionData.formatCity(cityMap.name)
if(standardCity){
saveData(standardCity, m.name, m.ad_code , ret);
def newRegion = RegionData.formatRegion(standardCity, m.name)
saveData(pro, standardCity, m.name, m.ad_code , ret, newRegion);
}else{
println cityMap.name +" ûÓбê×¼»¯"
}
@@ -102,7 +103,7 @@ class RegionPointCrawl {
def findCityCode(city){
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
def q = null;
if(city.contains("省") || city.endsWith("自治区")){
if(city.contains("省") || city.endsWith("自治区") || ["上海市","天津市","北京市","重庆市"].contains(city)){
q = new BasicDBObject("province", pattern);
}else{
q = new BasicDBObject("name", pattern);
@@ -156,18 +157,22 @@ class RegionPointCrawl {
}
}
def saveData(city, region, code, ret){
def saveData(pro, city, region, code, ret, formatRegion){
def q = new BasicDBObject()
q.put("code", code)
def dbo = POLYLINECOLL.findOne(q)
if(!dbo){
def o = new BasicDBObject()
o.put("province", pro)
o.put("city", city)
o.put("region", region)
o.put("code", code)
o.put("center", ret.center.get(0))
o.put("level", ret.level.get(0))
o.put("polyline", ret.polyline.get(0));
if(formatRegion){
o.put("fjw_region", formatRegion)
}
POLYLINECOLL.save(o);
}
}
@@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
//广州,佛山 重庆
@@ -204,8 +204,11 @@ public class BeiKeCrawlByChrome {
if(dealDate && dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
def dbo = findTrade(durl , "贝壳找房")
if (!dbo) {
parse(city, e, region, driver)
parse(city, e, region, driver, null)
}else if(dbo && !dbo.lift){
parse(city, e, region, driver, dbo._id)
}
}else{
stop = true;
@@ -224,7 +227,7 @@ public class BeiKeCrawlByChrome {
return detailUrls
}
def parse(city,e, region, driver){
def parse(city,e, region, driver, id){
try {
def detailUrl = e.select(".title a").attr("href")
@@ -289,20 +292,26 @@ public class BeiKeCrawlByChrome {
}
detail.source = "贝壳找房"
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡",""))
detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构",""))
detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡",""))
detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","")
detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属",""))
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限",""))
if(detail.total_price && detail.avg_price){
detail.url = detailUrl
@@ -323,8 +332,11 @@ public class BeiKeCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
if(id){
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
}else{
DEAL_PRICE_COLL.save(detail)
}
}
}
}catch(Exception ex){
@@ -453,9 +465,9 @@ public class BeiKeCrawlByChrome {
if(cityEles){
for(def e : cityEles){
def href = "https:"+e.attr("href")
// if (citys.contains(e.text())){
if (citys.contains(e.text())){
list << [e.text(), href]
// }
}
}
}
return list
@@ -483,7 +495,7 @@ public class BeiKeCrawlByChrome {
elemPassword.sendKeys("12qwaszx");
btn.submit(); // 点击行为;
Thread.sleep(20000)
Thread.sleep(10000)
}catch(Exception e){
Thread.sleep(15000)
}
@@ -1,6 +1,7 @@
package com.fangjia.script.gj.paimai
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.gj.ccb.FileUtils
@@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome {
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
static main(args) {
def task = new LianjiaCrawlByChrome();
@@ -297,16 +298,23 @@ public class LianjiaCrawlByChrome {
def dealDate = e.select(".dealDate").text()
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver)
if(dealDate){
if(dealDate.getTime()>=endDate.getTime()){
detailUrls << durl
def dbo = findTrade(durl , "链家在线")
if (!dbo) {
parse(city, e, regionText, driver, null)
}else if(dbo && !dbo.lift){
parse(city, e, regionText, driver, dbo._id)
}
}else{
stop = true;
break;
}
}else{
stop = true;
break;
println url +" 成交时间有误!"
}
}
if(page>=100){
stop = true;
@@ -320,14 +328,13 @@ public class LianjiaCrawlByChrome {
return detailUrls
}
def parse(city,e, regionText, driver){
def parse(city,e, regionText, driver, id){
try {
def price = e.select(".totalPrice .number").text()
if(price){
def detailUrl = e.select(".title a").attr("href")
driver.get(detailUrl)
def detailDoc = checkDocument(driver, detailUrl);
def title = e.select(".title a").text()
if (title) {
def detail = new BasicDBObject()
@@ -345,6 +352,15 @@ public class LianjiaCrawlByChrome {
if(detail.region){
detail.region = RegionData.formatRegion(city, detail.region)
}
if(!detail.region){
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
str = str.substring(0, str.indexOf("',"))
def locArrs = str.split(",")
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
if(gmap){
detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat)
}
}
def roomStr = arrs[1]
if (roomStr) {
@@ -386,20 +402,22 @@ public class LianjiaCrawlByChrome {
detail.source = "链家在线"
detail.url = detailUrl
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
def detailDiv = JQueryUtils.select(".introContent", detailDoc)
def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","")
if(liftStr){
detail.lift = DataFormatUtils.parseLift(liftStr)
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
}
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层",""))
detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","")
detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构",""))
detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属",""))
detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限",""))
detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯",""))
detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构",""))
detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代",""))
detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型",""))
def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层",""))
if(detail.total_price && detail.avg_price){
def houseInfo = e.select(".houseInfo").text()
@@ -417,7 +435,12 @@ public class LianjiaCrawlByChrome {
detail.create_date = new Date()
detail.m_date = new Date()
println detail
DEAL_PRICE_COLL.save(detail)
if(id){
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
}else{
DEAL_PRICE_COLL.save(detail)
}
}
}
}
@@ -547,9 +570,9 @@ public class LianjiaCrawlByChrome {
if(cityEles){
for(def e : cityEles) {
def href = e.attr("href")
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
cityList << [e.text(), href]
// }
}
}
}
return cityList
@@ -569,9 +592,11 @@ public class LianjiaCrawlByChrome {
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
elemUsername.sendKeys("15021384995");
// elemUsername.sendKeys("15021384995");
// elemPassword.sendKeys("12qwaszx");
elemPassword.sendKeys("12qwaszx");
elemUsername.sendKeys("16283166371");
elemPassword.sendKeys("Goojia123456");
btn.submit(); // 点击行为;
Thread.sleep(15000)
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
@@ -21,10 +22,11 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
}
@@ -105,12 +107,12 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['B']) {
// def dist = CommonDatautils.findDistrictById(d['B'])
// if(dist){
// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['G'] = gxdPrice
// }
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['I'] = gxdPrice
}
}
})
}
@@ -129,4 +131,22 @@ class AddPriceTest {
}
return price
}
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd"))
if(dbo){
// d['K'] = dbo.avg_price
d['K'] = dbo.avg_price_2b
}
}
}
})
}
}
@@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
if(dists){
for(def d : dists){
def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
def distObj = null
if(d.comm_id){
distObj = CommonDatautils.findDistrictById(d.comm_id)
}else{
distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
}
if(distObj && distObj._id){
FzDataUtils.converDistrict(distObj, null, null);
if(d.type && d.type==1){
FzDataUtils.removeById(d.comm_id)
}else{
FzDataUtils.converDistrict(distObj, null, null);
}
println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name
updateTaskStatus(d.id, 2);
}
@@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
def q = new BasicDBObject()
q.put("status", 5);//未处理
TASK_COLL.find(q).each {d->
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id]
}
return rets;
}
@@ -119,6 +119,13 @@ class FzDataUtils {
}
}
static def removeById(id){
def del_data = new BasicDBObject()
del_data.put("num_id", id)
DIST_COLL.remove(del_data)
}
static def saveDistrict(def m){
DBObject q = new BasicDBObject();
q.put("num_id",m.num_id)
@@ -294,4 +294,8 @@ class AddressUtil {
return result
}
public static void main(String[] args){
def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区")
}
}
@@ -52,7 +52,40 @@ class InsidePolygon {
return list;
}
public static List<Point> createPoint3(String points){
List<Point> alllist = new ArrayList<Point>();
if(points){
List<String[]> arrs = new ArrayList<String[]>();
if(points.contains("|")){
String[] array = points.split("\\|");
arrs.addAll(array)
}else{
arrs.add(points)
}
for(def arrStr : arrs){
List<Point> list = new ArrayList<Point>();
String[] array = arrStr.split(";");
for(int i =0 ;i<array.length-1;i++){
def pointArrs = array[i].split("[,|\\|]")
Point point = new Point();
double x = Double.parseDouble(pointArrs[0]);
double y = Double.parseDouble(pointArrs[1]);
point.setLon(x);
point.setLat(y);
list.add(point);
}
alllist << list
}
}
return alllist;
}
public static List<Map<String, Object>> getPoints(String points){
String[] array = points.split("-");
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
@@ -227,4 +260,16 @@ class InsidePolygon {
}
return false;
}
public static def isInArea(lng, lat, pointList){
def flag = false;
for(def pointArr : pointList){
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
if(flag){
break;
}
}
return flag;
}
}
@@ -24,7 +24,7 @@ class CommitteeLatLng {
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static void main(String[] args) {
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021))
dealPoi(new BasicDBObject("location", "").append("year", 2022))
//dealLocation()
executorService.shutdown()
}
@@ -71,7 +71,7 @@ class CommitteeLatLng {
if (amapPoi && amapPoi.size() > 0) {
def poi = amapPoi[0]
double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, name)
if (similarity <= 0.8) {
if (similarity <= 0.8 && !String.valueOf(poi.name).contains(name)) {
amapPoi = GaoDeUtils.searchPoi(city, address, area, "")
}
poi = amapPoi[0]
@@ -55,13 +55,17 @@ class GaoDeUtils {
* @return
*/
static String getGaodeKey(String type) {
Map<String, String> params = new HashMap<>();
params.put("type", type)
JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params)
if (json.status == 0 && json.result != null) {
return json.result
}
System.exit(1);
// Map<String, String> params = new HashMap<>();
// params.put("type", type)
// JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params)
// if (json.status == 0 && json.result != null) {
// return json.result
// }
// System.exit(1);
return "8796b9dfb3889b57c38e2fff1b7e51bd";
// 'eb69e4ad2a9cdd31f163a0ed32effe6e', '60f764d228d64addf390e28d68a01a35', '66e6771767ab293ace441553c36621c1',
// 'a63254b4a1b2e6f4bb11058e0b5d9fa2', '743ad872ad39f0a10665b28fa968af0b', '3b7c1d201c403bb416b63535ed3d0e98',
// 'a34894d151fffff94f9a33bae3d5565c', '389e0a3a155e8d28728c3faede4ebd4f'
}
/**
@@ -330,10 +334,10 @@ class GaoDeUtils {
params.page = "${page}".toString()
page++
params.key = this.getGaodeKey("v5/place/text")
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params);
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params);
if (json == null) {
params.key = this.getGaodeKey("v5/place/text")
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params);
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params);
}
if (json && json.info && json.info == "OK") {
count = json.getString("count").toInteger()
@@ -1,7 +1,9 @@
package com.fangjia.script.temp.shenjian.util
import net.sf.json.JSONObject
import org.apache.commons.httpclient.HttpClient
import org.apache.commons.httpclient.methods.PostMethod
import org.apache.commons.httpclient.methods.RequestEntity
import org.apache.commons.httpclient.methods.StringRequestEntity
/**
@@ -26,4 +28,42 @@ class HttpUtils {
}
return false
}
static def doPost2(String url, String jsonString){
HttpClient httpClient = new HttpClient()
PostMethod postMethod = new PostMethod(url)
String responseMsg = null;
try{
StringRequestEntity requestEntity = new StringRequestEntity(
jsonString,
"application/json",
"UTF-8");
postMethod.setRequestEntity(requestEntity);
httpClient.executeMethod(postMethod);
InputStream is = postMethod.getResponseBodyAsStream();
BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
StringBuffer sb = new StringBuffer("");
String str= "";
while((str = br.readLine()) != null){
sb.append(str);
}
responseMsg = sb.toString();
} catch (Exception ex) {
ex.printStackTrace();
} finally {
postMethod.releaseConnection();
}
return JSONObject.fromObject(responseMsg);
}
public String encodeHtml(String html) {
String mh=html;
try {
mh=new String(html.getBytes("iso-8859-1"));
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
return mh;
}
}
@@ -92,8 +92,10 @@ class DataStandardBatch extends BaseBatch{
cityMap.each{city,alias->
println "¿ªÊ¼¼ÓÔØ¡¾${city}¡¿"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// if(city == "¹ãÖÝ"){
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
districtMap.putAll(detailMap)
// }
}
while(flag){
@@ -833,7 +835,7 @@ class DataStandardBatch extends BaseBatch{
}
}
}
if(!dbo){
if(!dbo && isStandardRoad(copyObj.address)){
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
if(result.size()==1){
dbo = result[0];
@@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
}
@Override
void process(Object args) {
println SHORT_DATE
def citys = CommonDatautils.findAllCity();
def weekend = DateUtils.getFirstDayOfWeek(new Date());
@@ -28,7 +28,25 @@ class TradeDealPriceStat {
static main(args) {
def task = new TradeDealPriceStat()
task.run()
// task.run()
task.deleteRepeat()
}
def deleteRepeat(){
def q = new BasicDBObject()
q.put("source", "ÎÒ°®ÎÒ¼Ò")
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def set = new HashSet()
for(def c : cursor){
def tradeDate = DateUtils.formatDateByPattern(c.trade_date, "yyyyMMdd")
def key = c.city +"#"+c.region+"#"+c.district_name+"#"+c.avg_price+"#"+tradeDate
if(set.contains(key)){
DEAL_PRICE.remove(new BasicDBObject('_id', c._id))
println c._id
}else{
set << key
}
}
}
def run(){
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20230301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
startDate = nextDate
}
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303")
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20220901", "yyyyMMdd")
def endDate = DateUtil.parseDate("20221001", "yyyyMMdd")
def startDate = DateUtil.parseDate("20230201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20230301", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -0,0 +1,102 @@
package com.fangjia.service.bo;
import java.io.Serializable;
import java.util.Date;
/**
* @author zhaojixiang
* @since 2023/7/5
*/
public class BuildingRatioBo implements Serializable {
private static final long serialVersionUID = 1L;
private String id;
private String city_name;
private String region;
private String district;
private String building_name;
private Double ratio;
private Date c_date;
private Date m_date;
private String create_user;
private String note;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getCity_name() {
return city_name;
}
public void setCity_name(String city_name) {
this.city_name = city_name;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
public String getBuilding_name() {
return building_name;
}
public void setBuilding_name(String building_name) {
this.building_name = building_name;
}
public Double getRatio() {
return ratio;
}
public void setRatio(Double ratio) {
this.ratio = ratio;
}
public Date getC_date() {
return c_date;
}
public void setC_date(Date c_date) {
this.c_date = c_date;
}
public Date getM_date() {
return m_date;
}
public void setM_date(Date m_date) {
this.m_date = m_date;
}
public String getCreate_user() {
return create_user;
}
public void setCreate_user(String create_user) {
this.create_user = create_user;
}
public String getNote() {
return note;
}
public void setNote(String note) {
this.note = note;
}
}
@@ -0,0 +1,18 @@
package com.fangjia.service.mongo.bkoff;
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
import java.util.List;
import java.util.Map;
/**
* @author zhaojixiang
* @since 2023/7/5
*/
public interface BuildingRatioService {
public void save(BuildingRatio buildingRatio);
public void deleteById(String id);
public void saveOrUpdateByBuildingName(String city, String region, String district, String building_name, Map<String, Object> data);
public BuildingRatio findBuildingRatioById(String id);
public List<BuildingRatio> findAllByBuildingName(String city, String region, String district, String building_name);
}
@@ -0,0 +1,81 @@
package com.fangjia.service.mongo.bkoff;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.SortParam;
import com.fangjia.dal.mongo.dao.bkoff.BuildingRatioDao;
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
import com.fangjia.dal.sql.model.FjtUsers;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* @author zhaojixiang
* @since 2023/7/5
*/
public class BuildingRatioServiceImpl implements BuildingRatioService{
private BuildingRatioDao buildingRatioDao;
@Override
public void save(BuildingRatio buildingRatio) {
this.buildingRatioDao.save(buildingRatio);
}
@Override
public void deleteById(String id) {
this.buildingRatioDao.deleteById(id);
}
@Override
public void saveOrUpdateByBuildingName(String city, String region, String district, String building_name, Map<String, Object> data) {
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("city", city));
params.add(new QueryParam("region", region));
params.add(new QueryParam("district", district));
params.add(new QueryParam("building_name", building_name));
data.put("city", city);
data.put("region", region);
data.put("district", district);
data.put("building_name", building_name);
this.buildingRatioDao.updateByParams(data, params, true);
}
@Override
public BuildingRatio findBuildingRatioById(String id) {
return this.buildingRatioDao.findById(id);
}
@Override
public List<BuildingRatio> findAllByBuildingName(String city, String region, String district, String building_name) {
List<QueryParam> params = new ArrayList<QueryParam>();
if(StringUtils.isNotBlank(city)){
params.add(new QueryParam("city", city));
}
if(StringUtils.isNotBlank(region)){
params.add(new QueryParam("region", region));
}
if(StringUtils.isNotBlank(district)){
params.add(new QueryParam("district", district));
}
if(StringUtils.isNotBlank(building_name)){
params.add(new QueryParam("building_name", building_name));
}
List<SortParam> sortParams = new ArrayList<SortParam>();
sortParams.add(new SortParam("m_date", false));
return this.buildingRatioDao.findAllByParams(params, sortParams);
}
public BuildingRatioDao getBuildingRatioDao() {
return buildingRatioDao;
}
public void setBuildingRatioDao(BuildingRatioDao buildingRatioDao) {
this.buildingRatioDao = buildingRatioDao;
}
}
@@ -26,6 +26,7 @@ public class ApiUserBo implements Serializable {
private Date deadTime;
private String manager;
private String priceType;
private Integer isMonitor;
public String getPriceType() {
return priceType;
@@ -162,4 +163,12 @@ public class ApiUserBo implements Serializable {
public void setManager(String manager) {
this.manager = manager;
}
public Integer getIsMonitor() {
return isMonitor;
}
public void setIsMonitor(Integer isMonitor) {
this.isMonitor = isMonitor;
}
}