Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+4
@@ -102,6 +102,7 @@ public class ApiUserController {
|
||||
bo.setPurpose(apiUser.getPurpose());
|
||||
bo.setDeadTime(apiUser.getDeadTime());
|
||||
bo.setManager(apiUser.getManager());
|
||||
bo.setIsMonitor(apiUser.getIsMonitor());
|
||||
|
||||
apiUserBoList.add(bo);
|
||||
}
|
||||
@@ -183,6 +184,7 @@ public class ApiUserController {
|
||||
apiUser.setModificationDate(date);
|
||||
apiUser.setDeadTime(model.getDeadTime());
|
||||
apiUser.setManager(model.getManager());
|
||||
apiUser.setIsMonitor(model.getIsMonitor());
|
||||
this.apiUserService.save(apiUser);
|
||||
if(model.getEvaluateRate() != null && model.getEvaluateRate() != 0){
|
||||
ApiUserCfg apiUserCfg = new ApiUserCfg();
|
||||
@@ -222,6 +224,7 @@ public class ApiUserController {
|
||||
bo.setEmail(apiUser.getEmail());
|
||||
bo.setDeadTime(apiUser.getDeadTime());
|
||||
bo.setManager(apiUser.getManager());
|
||||
bo.setIsMonitor(apiUser.getIsMonitor());
|
||||
ApiUserCfg apiUserCfg_rate = apiUserCfgService.findByUserId(apiUser.getId(), ApiUserCfg.ApiUserCfgKey.EVALUATE_RATE);
|
||||
if(apiUserCfg_rate != null){
|
||||
bo.setEvaluateRate((Double) apiUserCfg_rate.getCfgValue());
|
||||
@@ -272,6 +275,7 @@ public class ApiUserController {
|
||||
dataApiUser.put("purpose", model.getPurpose());
|
||||
dataApiUser.put("deadTime", model.getDeadTime());
|
||||
dataApiUser.put("manager", model.getManager());
|
||||
dataApiUser.put("isMonitor", model.getIsMonitor());
|
||||
this.apiUserService.update(dataApiUser, userId);
|
||||
|
||||
//ApiUserCfg.ApiUserCfgKey.EVALUATE_RATE
|
||||
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
package com.fangjia.bkoff.controllers;
|
||||
|
||||
import com.fangjia.bkoff.model.ApiUserModel;
|
||||
import com.fangjia.bkoff.model.BuildingRatioModel;
|
||||
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
|
||||
import com.fangjia.dal.sql.model.User;
|
||||
import com.fangjia.service.bo.BuildingRatioBo;
|
||||
import com.fangjia.service.mongo.bkoff.BuildingRatioService;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.net.URLEncoder;
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* @author zhaojixiang
|
||||
* @since 2023/7/5
|
||||
*/
|
||||
public class BuildingRatioController {
|
||||
|
||||
private BuildingRatioService buildingRatioService;
|
||||
|
||||
/**
|
||||
* 新建楼栋系数信息
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object doAddAction(BuildingRatioModel model) throws Exception {
|
||||
try {
|
||||
if(checkBuildingNameAction(model)){
|
||||
model.redirect("/buildingRatio/add?msg=" + URLEncoder.encode(model.getMsg(), "UTF-8"));
|
||||
return model;
|
||||
}
|
||||
Date date = new Date();
|
||||
BuildingRatio buildingRatio = new BuildingRatio();
|
||||
buildingRatio.setCity(model.getCity_name());
|
||||
buildingRatio.setRegion(model.getRegion());
|
||||
buildingRatio.setDistrict(model.getDistrict());
|
||||
buildingRatio.setBuilding_name(model.getBuilding_name());
|
||||
buildingRatio.setRatio(model.getRatio());
|
||||
buildingRatio.setC_date(date);
|
||||
buildingRatio.setM_date(date);
|
||||
buildingRatio.setCreate_user(model.getCreate_user());
|
||||
buildingRatio.setNote(model.getNote());
|
||||
this.buildingRatioService.save(buildingRatio);
|
||||
|
||||
model.setMsg("新建成功!");
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
model.setMsg("新建失败!");
|
||||
}
|
||||
model.redirect("/buildingRatio/add?msg=" + URLEncoder.encode(model.getMsg(), "UTF-8"));
|
||||
return model;
|
||||
}
|
||||
|
||||
public Object doDeleteAction(BuildingRatioModel model){
|
||||
if(StringUtils.isNotBlank(model.getId())){
|
||||
this.buildingRatioService.deleteById(model.getId());
|
||||
}
|
||||
model.setViewType("redirect");
|
||||
model.setViewParam("location","/buildingRatio/list");
|
||||
return model;
|
||||
}
|
||||
|
||||
/**
|
||||
* 修改楼栋系数信息
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object updateAction(BuildingRatioModel model) throws Exception {
|
||||
Map<String, Object> data_buildingRatio = new HashMap<String, Object>();
|
||||
|
||||
data_buildingRatio.put("id", model.getId());
|
||||
data_buildingRatio.put("city", model.getCity_name());
|
||||
data_buildingRatio.put("region", model.getRegion());
|
||||
data_buildingRatio.put("district", model.getDistrict());
|
||||
data_buildingRatio.put("building_name", model.getBuilding_name());
|
||||
data_buildingRatio.put("ratio", model.getRatio());
|
||||
data_buildingRatio.put("m_date", new Date());
|
||||
data_buildingRatio.put("create_user", model.getCreate_user());
|
||||
data_buildingRatio.put("note", model.getNote());
|
||||
|
||||
this.buildingRatioService.saveOrUpdateByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name(), data_buildingRatio);
|
||||
|
||||
model.setViewType("redirect");
|
||||
model.setViewParam("location", "/buildingRatio/list");
|
||||
|
||||
return model;
|
||||
}
|
||||
|
||||
public Object listAction(BuildingRatioModel model) {
|
||||
|
||||
List<BuildingRatio> buildingRatioList = this.buildingRatioService.findAllByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name());
|
||||
|
||||
List<BuildingRatioBo> buildingRatioBoList = new ArrayList<BuildingRatioBo>();
|
||||
|
||||
for (BuildingRatio buildingRatio : buildingRatioList) {
|
||||
BuildingRatioBo bo = new BuildingRatioBo();
|
||||
|
||||
bo.setId(buildingRatio.getId());
|
||||
bo.setCity_name(buildingRatio.getCity());
|
||||
bo.setRegion(buildingRatio.getRegion());
|
||||
bo.setDistrict(buildingRatio.getDistrict());
|
||||
bo.setBuilding_name(buildingRatio.getBuilding_name());
|
||||
bo.setRatio(buildingRatio.getRatio());
|
||||
bo.setC_date(buildingRatio.getC_date());
|
||||
bo.setM_date(buildingRatio.getM_date());
|
||||
bo.setCreate_user(buildingRatio.getCreate_user());
|
||||
bo.setNote(buildingRatio.getNote());
|
||||
|
||||
buildingRatioBoList.add(bo);
|
||||
}
|
||||
|
||||
model.setBuildingRatioBoList(buildingRatioBoList);
|
||||
return model;
|
||||
}
|
||||
|
||||
public Object addAction(BuildingRatioModel model) {
|
||||
return model;
|
||||
}
|
||||
|
||||
/**
|
||||
* 编辑api用户信息
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object editAction(BuildingRatioModel model) {
|
||||
BuildingRatio buildingRatio = this.buildingRatioService.findBuildingRatioById(model.getId());
|
||||
|
||||
BuildingRatioBo bo = new BuildingRatioBo();
|
||||
bo.setId(buildingRatio.getId());
|
||||
bo.setCity_name(buildingRatio.getCity());
|
||||
bo.setRegion(buildingRatio.getRegion());
|
||||
bo.setDistrict(buildingRatio.getDistrict());
|
||||
bo.setBuilding_name(buildingRatio.getBuilding_name());
|
||||
bo.setRatio(buildingRatio.getRatio());
|
||||
bo.setCreate_user(buildingRatio.getCreate_user());
|
||||
bo.setNote(buildingRatio.getNote());
|
||||
|
||||
model.setBuildingRatioBo(bo);
|
||||
if (StringUtils.isNotBlank(model.getMsg()) && model.getMsg().equals("success")) {
|
||||
model.setMsg("更新成功!");
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
public boolean checkBuildingNameAction(BuildingRatioModel model) {
|
||||
model.setViewType("json");
|
||||
model.setViewParam("config", null);
|
||||
List<BuildingRatio> buildingRatioList = buildingRatioService.findAllByBuildingName(model.getCity_name(),model.getRegion(),model.getDistrict(),model.getBuilding_name());
|
||||
if (buildingRatioList.size()>0) {
|
||||
model.setMsg("此楼栋名称已经存在!");
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public BuildingRatioService getBuildingRatioService() {
|
||||
return buildingRatioService;
|
||||
}
|
||||
|
||||
public void setBuildingRatioService(BuildingRatioService buildingRatioService) {
|
||||
this.buildingRatioService = buildingRatioService;
|
||||
}
|
||||
}
|
||||
@@ -38,6 +38,7 @@ public class ApiUserModel extends BaseModel {
|
||||
private Integer creditLevelStatus;
|
||||
private String priceType;
|
||||
private Map<String,Double> priceRatingMap;
|
||||
private Integer isMonitor;
|
||||
|
||||
public String getPriceType() {
|
||||
return priceType;
|
||||
@@ -606,4 +607,12 @@ public class ApiUserModel extends BaseModel {
|
||||
public void setManager(String manager) {
|
||||
this.manager = manager;
|
||||
}
|
||||
|
||||
public Integer getIsMonitor() {
|
||||
return isMonitor;
|
||||
}
|
||||
|
||||
public void setIsMonitor(Integer isMonitor) {
|
||||
this.isMonitor = isMonitor;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
package com.fangjia.bkoff.model;
|
||||
|
||||
import com.fangjia.service.bo.BuildingRatioBo;
|
||||
import org.springframework.format.annotation.DateTimeFormat;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* @author zhaojixiang
|
||||
* @since 2023/7/5
|
||||
*/
|
||||
public class BuildingRatioModel extends BaseModel{
|
||||
|
||||
private String id;
|
||||
private String city_name;
|
||||
private String region;
|
||||
private String district;
|
||||
private String building_name;
|
||||
private String create_user;
|
||||
private String note;
|
||||
private Double ratio;
|
||||
private String msg;
|
||||
private List<BuildingRatioBo> buildingRatioBoList;
|
||||
private BuildingRatioBo buildingRatioBo;
|
||||
@DateTimeFormat(pattern = "yyyy-MM-dd")
|
||||
private Date c_date;
|
||||
@DateTimeFormat(pattern = "yyyy-MM-dd")
|
||||
private Date m_date;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getCity_name() {
|
||||
return city_name;
|
||||
}
|
||||
|
||||
public void setCity_name(String city_name) {
|
||||
this.city_name = city_name;
|
||||
}
|
||||
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
|
||||
public void setRegion(String region) {
|
||||
this.region = region;
|
||||
}
|
||||
|
||||
public String getDistrict() {
|
||||
return district;
|
||||
}
|
||||
|
||||
public void setDistrict(String district_name) {
|
||||
this.district = district_name;
|
||||
}
|
||||
|
||||
public String getBuilding_name() {
|
||||
return building_name;
|
||||
}
|
||||
|
||||
public void setBuilding_name(String building_name) {
|
||||
this.building_name = building_name;
|
||||
}
|
||||
|
||||
public Double getRatio() {
|
||||
return ratio;
|
||||
}
|
||||
|
||||
public void setRatio(Double ratio) {
|
||||
this.ratio = ratio;
|
||||
}
|
||||
|
||||
public Date getC_date() {
|
||||
return c_date;
|
||||
}
|
||||
|
||||
public void setC_date(Date c_date) {
|
||||
this.c_date = c_date;
|
||||
}
|
||||
|
||||
public Date getM_date() {
|
||||
return m_date;
|
||||
}
|
||||
|
||||
public void setM_date(Date m_date) {
|
||||
this.m_date = m_date;
|
||||
}
|
||||
|
||||
public List<BuildingRatioBo> getBuildingRatioBoList() {
|
||||
return buildingRatioBoList;
|
||||
}
|
||||
|
||||
public void setBuildingRatioBoList(List<BuildingRatioBo> buildingRatioBoList) {
|
||||
this.buildingRatioBoList = buildingRatioBoList;
|
||||
}
|
||||
|
||||
public BuildingRatioBo getBuildingRatioBo() {
|
||||
return buildingRatioBo;
|
||||
}
|
||||
|
||||
public void setBuildingRatioBo(BuildingRatioBo buildingRatioBo) {
|
||||
this.buildingRatioBo = buildingRatioBo;
|
||||
}
|
||||
|
||||
public String getMsg() {
|
||||
return msg;
|
||||
}
|
||||
|
||||
public void setMsg(String msg) {
|
||||
this.msg = msg;
|
||||
}
|
||||
|
||||
public String getCreate_user() {
|
||||
return create_user;
|
||||
}
|
||||
|
||||
public void setCreate_user(String create_user) {
|
||||
this.create_user = create_user;
|
||||
}
|
||||
|
||||
public String getNote() {
|
||||
return note;
|
||||
}
|
||||
|
||||
public void setNote(String note) {
|
||||
this.note = note;
|
||||
}
|
||||
}
|
||||
@@ -57,13 +57,20 @@
|
||||
<td>账户用途</td>
|
||||
<td>
|
||||
<select name="purpose">
|
||||
<option value="" >请选择</option>
|
||||
<option value="" >请选择</option>
|
||||
<option value="1" >API</option>
|
||||
<option value="2" >PDS</option>
|
||||
<option value="4" >房生活</option>
|
||||
</select>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>是否监控</td>
|
||||
<td>
|
||||
<input type="radio" name="isMonitor" value="0" checked="checked"/>不监控
|
||||
<input type="radio" name="isMonitor" value="1" style="margin-left: 15px;"/>监控
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Email</td>
|
||||
<td><input name="email" value="${email!}" /></td>
|
||||
|
||||
@@ -72,6 +72,13 @@
|
||||
</select>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>是否监控</td>
|
||||
<td>
|
||||
<input type="radio" <#if apiUserBo?? && apiUserBo.isMonitor?? && apiUserBo.isMonitor == 0>checked="checked"</#if> name="isMonitor" value="0" />不监控
|
||||
<input type="radio" <#if apiUserBo?? && apiUserBo.isMonitor?? && apiUserBo.isMonitor == 1>checked="checked"</#if> name="isMonitor" value="1" style="margin-left: 15px;"/>监控
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Email</td>
|
||||
<td><input name="email" value="${apiUserBo.email!}"/></td>
|
||||
|
||||
@@ -53,6 +53,9 @@
|
||||
<th align="center" width="60px">
|
||||
账户用途
|
||||
</th>
|
||||
<th align="center" width="60px">
|
||||
是否监控
|
||||
</th>
|
||||
<th align="center" width="120px">
|
||||
到期时间
|
||||
</th>
|
||||
@@ -114,6 +117,13 @@
|
||||
房生活
|
||||
</#if>
|
||||
</td>
|
||||
<td>
|
||||
<#if bo.isMonitor==1>
|
||||
是
|
||||
<#else>
|
||||
否
|
||||
</#if>
|
||||
</td>
|
||||
<td align="center">
|
||||
<#if bo.deadTime??>
|
||||
${bo.deadTime?string("yyyy年MM月dd日")}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
|
||||
<#compress>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||||
<title>楼栋系数新增</title>
|
||||
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
|
||||
<#include "/common_js.ftl" />
|
||||
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<#include "/layout/header.ftl" />
|
||||
<div class="main-content">
|
||||
<font color="red">${msg!''}</font>
|
||||
<div class="menuButton">
|
||||
<a href="list" class="list">楼栋系数列表</a>
|
||||
</div>
|
||||
<hr/>
|
||||
<div class="list">
|
||||
<table width="50%">
|
||||
<thead>
|
||||
<tr>
|
||||
<th align="center" width="150px;">
|
||||
栏位
|
||||
</th>
|
||||
<th align="center">
|
||||
值
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<form id="addForm" action="doAdd" method="post" >
|
||||
<tr>
|
||||
<td><b>城市</b></td>
|
||||
<td><input name="city_name" value="${city_name!}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><b>区域</b></td>
|
||||
<td><input id="region" name="region" value="${region!}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><b>小区名称</b></td>
|
||||
<td><input name="district" type="text" value="${district!}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>楼栋名称</td>
|
||||
<td><input name="building_name" value="${building_name!}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>楼栋系数</td>
|
||||
<td><input name="ratio" value="${ratio!}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>创建人</td>
|
||||
<td><input name="create_user" value="${session.realName!""}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>备注</td>
|
||||
<td><input name="note" value="${note!}" /></td>
|
||||
</tr>
|
||||
</form>
|
||||
<tr>
|
||||
<td colspan="2"><input onclick="submitForm();" type="submit" value="新增"/></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<script type="text/javascript">
|
||||
function submitForm(){
|
||||
var city_name = $("input[name='city_name']").val();
|
||||
var region = $("input[name='region']").val();
|
||||
var district = $("input[name='district']").val();
|
||||
var building_name = $("input[name='building_name']").val();
|
||||
var ratio = $("input[name='ratio']").val();
|
||||
var create_user = $("input[name='create_user']").val();
|
||||
|
||||
if(typeof(city_name) == "undefined" || city_name == ""){
|
||||
alert("城市不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(region) == "undefined" || region == ""){
|
||||
alert("区域不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(district) == "undefined" || district == ""){
|
||||
alert("小区名称不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(building_name) == "undefined" || building_name == ""){
|
||||
alert("楼栋名称不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(ratio) == "undefined" || ratio == ""){
|
||||
alert("楼栋系数不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(create_user) == "undefined" || create_user == ""){
|
||||
alert("创建人不能为空");
|
||||
return false;
|
||||
}
|
||||
$("#addForm").submit();
|
||||
}
|
||||
var getErrorValidate= function(mess){
|
||||
var _html='<span class=\"wrong\">';
|
||||
_html=_html+mess;
|
||||
_html=_html+'</span>';
|
||||
return _html;
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
</#compress>
|
||||
@@ -0,0 +1,113 @@
|
||||
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
|
||||
<#compress>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
|
||||
<title>楼栋系数信息修改</title>
|
||||
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}"/>
|
||||
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
|
||||
<#include "/common_js.ftl" />
|
||||
<script type="text/javascript">
|
||||
</script>
|
||||
|
||||
|
||||
</head>
|
||||
<body>
|
||||
<#include "/layout/header.ftl" />
|
||||
<div class="main-content">
|
||||
<font color="red">${msg!''}</font>
|
||||
<div class="menuButton">
|
||||
<a href="list" class="list">楼栋系数列表</a><a href="add" class="create">新建楼栋系数</a>
|
||||
</div>
|
||||
<hr/>
|
||||
<div class="list">
|
||||
<table width="50%">
|
||||
<thead>
|
||||
<tr>
|
||||
<th align="center" width="10%">
|
||||
栏位
|
||||
</th>
|
||||
<th align="center" width="30%">
|
||||
值
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<form id="submitForm" action="update" method="post">
|
||||
<tr>
|
||||
<td>城市</td>
|
||||
<td><input name="city_name" value="${buildingRatioBo.city_name!}"/></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>区域</td>
|
||||
<td><input name="region" value="${buildingRatioBo.region!}"/></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>小区名称</td>
|
||||
<td><input name="district" value="${buildingRatioBo.district!}"/></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>楼栋名称</td>
|
||||
<input name="id" value="${buildingRatioBo.id!}" type="hidden"/>
|
||||
<td><input name="building_name" value="${buildingRatioBo.building_name!}"/></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>楼栋系数</td>
|
||||
<td><input name="ratio" value="${buildingRatioBo.ratio!}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>创建人</td>
|
||||
<td><input name="create_user" value="${buildingRatioBo.create_user!}" /></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>备注</td>
|
||||
<td><input name="note" value="${buildingRatioBo.note!}" /></td>
|
||||
</tr>
|
||||
</form>
|
||||
<tr>
|
||||
<td></td>
|
||||
<td><input onclick="submitForm();" type="reset" value="修改"/></td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<script type="text/javascript">
|
||||
function submitForm() {
|
||||
var city_name = $("input[name='city_name']").val();
|
||||
var region = $("input[name='region']").val();
|
||||
var district = $("input[name='district']").val();
|
||||
var building_name = $("input[name='building_name']").val();
|
||||
var ratio = $("input[name='ratio']").val();
|
||||
var create_user = $("input[name='create_user']").val();
|
||||
|
||||
if(typeof(city_name) == "undefined" || city_name == ""){
|
||||
alert("城市不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(region) == "undefined" || region == ""){
|
||||
alert("区域不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(district) == "undefined" || district == ""){
|
||||
alert("小区名称不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(building_name) == "undefined" || building_name == ""){
|
||||
alert("楼栋名称不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(ratio) == "undefined" || ratio == ""){
|
||||
alert("楼栋系数不能为空");
|
||||
return false;
|
||||
}
|
||||
if(typeof(create_user) == "undefined" || create_user == ""){
|
||||
alert("创建人不能为空");
|
||||
return false;
|
||||
}
|
||||
$("#submitForm").attr("action", "update");
|
||||
$("#submitForm").submit();
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
</#compress>
|
||||
@@ -0,0 +1,120 @@
|
||||
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
|
||||
<#compress>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
|
||||
<title>均价管理-楼栋系数管理</title>
|
||||
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
|
||||
<#include "/common_js.ftl" />
|
||||
</head>
|
||||
<body>
|
||||
<#include "/layout/header.ftl" />
|
||||
|
||||
<div>
|
||||
<form action="../buildingRatio/list" id="_search_ratio_form">
|
||||
城市:<input type="text" name="city_name" value="${city_name!}" />
|
||||
区域:<input type="text" name="region" value="${region!}" />
|
||||
小区:<input type="text" name="district" value="${district!}" />
|
||||
<input type="submit" value="查询" />
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="main-content">
|
||||
<font color="red">${msg!''}</font>
|
||||
<div class="menuButton">
|
||||
<a href="list" class="list">楼栋系数列表</a><a href="add" class="create">新建楼栋系数</a>
|
||||
</div>
|
||||
<hr/>
|
||||
<div class="list">
|
||||
<div style="padding-left:5px;">
|
||||
共有楼栋系数<b>${buildingRatioBoList?size}</b>个
|
||||
</div>
|
||||
<table width="100%">
|
||||
<thead>
|
||||
<tr>
|
||||
<th align="center" width="55px">
|
||||
ID
|
||||
</th>
|
||||
<th align="center" width="100px">
|
||||
城市
|
||||
</th>
|
||||
<th align="center" width="60px">
|
||||
区域
|
||||
</th>
|
||||
<th align="center" width="100px">
|
||||
小区名称
|
||||
</th>
|
||||
<th align="center" width="60px">
|
||||
楼栋名称
|
||||
</th>
|
||||
<th align="center" width="60px">
|
||||
楼栋系数
|
||||
</th>
|
||||
<th align="center" width="60px">
|
||||
创建人
|
||||
</th>
|
||||
<th align="center" width="200px">
|
||||
创建日期
|
||||
</th>
|
||||
<th align="center" width="200px">
|
||||
修改日期
|
||||
</th>
|
||||
<th align="center" width="100px">
|
||||
备注
|
||||
</th>
|
||||
<th align="center">
|
||||
操作
|
||||
</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<#if (buildingRatioBoList??&&buildingRatioBoList?size>0)>
|
||||
<#list buildingRatioBoList as bo>
|
||||
<tr>
|
||||
<td align="center">
|
||||
${bo.id!}
|
||||
</td>
|
||||
<td align="center">
|
||||
${bo.city_name!}
|
||||
</td>
|
||||
<td align="center">
|
||||
${bo.region!}
|
||||
</td>
|
||||
<td align="center">
|
||||
${bo.district!}
|
||||
</td>
|
||||
<td align="center">
|
||||
${bo.building_name!}
|
||||
</td>
|
||||
<td align="center">
|
||||
${bo.ratio!}
|
||||
</td>
|
||||
<td align="center">
|
||||
${bo.create_user!}
|
||||
</td>
|
||||
<td align="center">
|
||||
<#if bo.c_date??>
|
||||
${bo.c_date?string("yyyy年MM月dd日 HH:mm:ss")}
|
||||
</#if>
|
||||
</td>
|
||||
<td align="center">
|
||||
<#if bo.m_date??>
|
||||
${bo.m_date?string("yyyy年MM月dd日 HH:mm:ss")}
|
||||
</#if>
|
||||
</td>
|
||||
<td align="center">
|
||||
${bo.note!}
|
||||
</td>
|
||||
<td align="center">
|
||||
<a href="../buildingRatio/edit?id=${bo.id!}">编辑</a>
|
||||
<a href="../buildingRatio/doDelete?id=${bo.id}" onclick="return confirm('确定要删除 ${bo.city_name!}-${bo.region!}-${bo.district!}-${bo.building_name!}?')">删除</a>
|
||||
</td>
|
||||
</tr>
|
||||
</#list>
|
||||
</#if>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
</#compress>
|
||||
@@ -90,6 +90,7 @@
|
||||
<ul>
|
||||
<@menuitem uri='/avgPriceUpdate/doCreatTask' text="住宅均价管理" />
|
||||
<@menuitem uri='/averagerConfig/list' text="住宅系数管理" />
|
||||
<@menuitem uri='/buildingRatio/list' text="楼栋系数管理" />
|
||||
<@menuitem uri='/avgPriceUpdate/listQR' text="价格质量跟踪" />
|
||||
<@menuitem uri='/avgPriceUpdate/listSQR' text="样本质量跟踪" />
|
||||
<@menuitem uri='/comparePrice/listQR' text="调查数据跟踪" />
|
||||
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package com.fangjia.dal.mongo.dao.bkoff;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.model.bkoff.AveragerConfig;
|
||||
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
|
||||
|
||||
/**
|
||||
* @author zhaojixiang
|
||||
* @since 2023/7/5
|
||||
*/
|
||||
public interface BuildingRatioDao extends MongoDao<BuildingRatio> {
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
package com.fangjia.dal.mongo.dao.bkoff;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.mongo.model.bkoff.AveragerConfig;
|
||||
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
|
||||
|
||||
/**
|
||||
* @author zhaojixiang
|
||||
* @since 2023/7/5
|
||||
*/
|
||||
public class BuildingRatioDaoImpl extends MongoDaoImpl<BuildingRatio> implements BuildingRatioDao {
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package com.fangjia.dal.mongo.model.bkoff;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.Column;
|
||||
import javax.persistence.Entity;
|
||||
import javax.persistence.GeneratedValue;
|
||||
import javax.persistence.Id;
|
||||
import javax.persistence.Table;
|
||||
|
||||
import org.springframework.format.annotation.DateTimeFormat;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* @author zhaojixiang
|
||||
* @since 2023/7/5
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "business.building_ratio")
|
||||
public class BuildingRatio implements MongoEntity {
|
||||
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String district;
|
||||
private String building_name;
|
||||
|
||||
private Double ratio;
|
||||
@DateTimeFormat(pattern = "yyyy-MM-dd")
|
||||
private Date c_date;
|
||||
@DateTimeFormat(pattern = "yyyy-MM-dd")
|
||||
private Date m_date;
|
||||
private String create_user;
|
||||
private String note;
|
||||
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@Column(name = "_id")
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
@Column(name = "city")
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
@Column(name = "region")
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
public void setRegion(String region) {
|
||||
this.region = region;
|
||||
}
|
||||
@Column(name = "district")
|
||||
public String getDistrict() {
|
||||
return district;
|
||||
}
|
||||
public void setDistrict(String district) {
|
||||
this.district = district;
|
||||
}
|
||||
@Column(name = "building_name")
|
||||
public String getBuilding_name() {
|
||||
return building_name;
|
||||
}
|
||||
public void setBuilding_name(String building_name) {
|
||||
this.building_name = building_name;
|
||||
}
|
||||
@Column(name = "ratio")
|
||||
public Double getRatio() {
|
||||
return ratio;
|
||||
}
|
||||
public void setRatio(Double ratio) {
|
||||
this.ratio = ratio;
|
||||
}
|
||||
@Column(name = "c_date")
|
||||
public Date getC_date() {
|
||||
return c_date;
|
||||
}
|
||||
public void setC_date(Date c_date) {
|
||||
this.c_date = c_date;
|
||||
}
|
||||
@Column(name = "m_date")
|
||||
public Date getM_date() {
|
||||
return m_date;
|
||||
}
|
||||
public void setM_date(Date m_date) {
|
||||
this.m_date = m_date;
|
||||
}
|
||||
@Column(name = "create_user")
|
||||
public String getCreate_user() {
|
||||
return create_user;
|
||||
}
|
||||
|
||||
public void setCreate_user(String create_user) {
|
||||
this.create_user = create_user;
|
||||
}
|
||||
@Column(name = "note")
|
||||
public String getNote() {
|
||||
return note;
|
||||
}
|
||||
|
||||
public void setNote(String note) {
|
||||
this.note = note;
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,8 @@ public class ApiUser implements SqlEntity {
|
||||
|
||||
private String manager;
|
||||
|
||||
private Integer isMonitor;
|
||||
|
||||
@Id
|
||||
@Column(name = "ID")
|
||||
public long getId() {
|
||||
@@ -175,4 +177,12 @@ public class ApiUser implements SqlEntity {
|
||||
public void setManager(String manager) {
|
||||
this.manager = manager;
|
||||
}
|
||||
@Column(name = "IS_MONITOR")
|
||||
public Integer getIsMonitor() {
|
||||
return isMonitor;
|
||||
}
|
||||
|
||||
public void setIsMonitor(Integer isMonitor) {
|
||||
this.isMonitor = isMonitor;
|
||||
}
|
||||
}
|
||||
|
||||
+21
-3
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳5月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -51,7 +51,19 @@ for(def d : datas){
|
||||
map.region = d['B']
|
||||
def roomStr = d['E']
|
||||
if(roomStr){
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
if(roomStr.contains("-")){
|
||||
def arrs = roomStr.split("-")
|
||||
try{
|
||||
map.room = arrs[0].toString().toInteger()
|
||||
}catch(Exception e){
|
||||
println "room error"
|
||||
}
|
||||
map.hall = arrs[1].toString().toInteger()
|
||||
map.toilet = arrs[3].toString().toInteger()
|
||||
}else{
|
||||
map.room = roomStr.toString().toDouble().toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
map.area = d['F'].toString().toDouble()
|
||||
@@ -59,7 +71,13 @@ for(def d : datas){
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
def trade_date = d['J']
|
||||
map.trade_date = trade_date
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
}
|
||||
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
map.m_date = new Date()
|
||||
|
||||
+2
-3
@@ -77,9 +77,8 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
println "txt 生成->导Oracle"
|
||||
|
||||
|
||||
// delPgData()
|
||||
// saveToPgSql(rows)
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
|
||||
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
|
||||
|
||||
+23
-11
@@ -33,7 +33,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM"));
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
@@ -61,17 +61,19 @@ class BigdataDealBatch extends BaseBatch{
|
||||
public void process(Object args) {
|
||||
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
|
||||
// def citys = CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
// def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
if(args && args.size()>=2){
|
||||
curDate = args[1].toString().toInteger()
|
||||
}
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
def allCitys = ["福州"] //CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
// if(!ccbCitys.contains(city)){
|
||||
addDealPriceMatch(city);
|
||||
addDealPriceMatch(city);
|
||||
// }
|
||||
}
|
||||
|
||||
|
||||
// def rows = []
|
||||
// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
@@ -649,16 +651,21 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
for(def i=0;i<trades.size();i++){
|
||||
def t = trades[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
final def t = trades[i]
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
count_.addAndGet(1)
|
||||
try{
|
||||
if(!t.match_id){
|
||||
def region = RegionData.formatRegion(t.city, t.region)
|
||||
def rMap = null
|
||||
if(region) {
|
||||
if(t.house_use && t.house_use.contains("商住")){
|
||||
if(!t.name.contains("商住")){
|
||||
t.name = t.name +"商住楼"
|
||||
}
|
||||
}
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
||||
if(!rMap && t.address){
|
||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
@@ -668,7 +675,12 @@ class BigdataDealBatch extends BaseBatch{
|
||||
}
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else {
|
||||
if (rMap) {
|
||||
println "区域不同"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
println count_.get()+" "+t+" "+ rMap
|
||||
}
|
||||
@@ -678,9 +690,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
// }
|
||||
// }
|
||||
// );
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
|
||||
+1
-1
@@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230518, 20230619, houseIdsfilter);
|
||||
//测试通过匹配接口配房源效果
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
|
||||
}
|
||||
|
||||
+3
-3
@@ -428,7 +428,7 @@ class CcbCommonUtils {
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
@@ -617,14 +617,14 @@ class CcbCommonUtils {
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate))
|
||||
q.put("match_id", new BasicDBObject('$exists', false))
|
||||
// q.put("district_name","国贸天琴湾")
|
||||
// q.put("district_name","中海寰宇天下")
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def total = dbcursor.count()
|
||||
def rets = new HashSet()
|
||||
def lists = []
|
||||
println city +" "+total
|
||||
for(def d : dbcursor){
|
||||
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id]
|
||||
lists << ["_id":d._id,"city":d.city, "region":d.region, "name":d.district_name, "address":d.address, "match_id":d.match_id, "house_use":d.house_use,"lift_household":d.lift_household,"lift":d.lift]
|
||||
}
|
||||
return lists
|
||||
}
|
||||
|
||||
+44
-5
@@ -28,11 +28,12 @@ class DealPriceDataProcess {
|
||||
def task = new DealPriceDataProcess()
|
||||
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
|
||||
// task.addLianJiaPrice();
|
||||
// task.deleteThirdPartyData()//删除第三方
|
||||
task.deleteThirdPartyData()//删除第三方
|
||||
// task.delDealByPe()
|
||||
// task.delDealData()
|
||||
|
||||
task.exportThirdParty()
|
||||
// task.exportThirdParty()
|
||||
// task.statPpe()
|
||||
}
|
||||
|
||||
def delDealByPe(){
|
||||
@@ -379,7 +380,7 @@ class DealPriceDataProcess {
|
||||
def results = []
|
||||
results << ['城市id','城市名称','第三方id','区域','小区id','小区名称','计算类型','计算价格','成交价格','PPE']
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty3(city, 202305)
|
||||
def rets = findPeGteThirdParty3(city, 202306)
|
||||
for(def map : rets){
|
||||
results << [map.cityid, map.city, map.data_id, map.region, map.comm_id, map.district_name, map.type, map.clc_price, map.price, map.ppe]
|
||||
}
|
||||
@@ -391,12 +392,12 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
def citys = CcbCommonUtils.citys_all
|
||||
def month = 202304
|
||||
def month = 202305
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def city : citys) {
|
||||
def rets = findPeGteThirdParty2(city, 202305)
|
||||
def rets = findPeGteThirdParty2(city, 202306)
|
||||
def total = 0
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
@@ -490,5 +491,43 @@ class DealPriceDataProcess {
|
||||
return count
|
||||
}
|
||||
|
||||
def statPpe(){
|
||||
def path = "C:\\Users\\gaojuan\\Desktop\\估值数据统计.xlsx"
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
|
||||
def deal_pes = []
|
||||
def third_pes = []
|
||||
for(def d : dates){
|
||||
def map = findPeGteCity(row['B'], d)
|
||||
if(map.deal_pe){
|
||||
deal_pes << map.deal_pe
|
||||
}
|
||||
if(map.third_pe){
|
||||
third_pes << map.third_pe
|
||||
}
|
||||
}
|
||||
if(deal_pes){
|
||||
row['F'] = deal_pes.sum()/deal_pes.size()
|
||||
}
|
||||
if(third_pes){
|
||||
row['G'] = third_pes.sum()/third_pes.size()
|
||||
}
|
||||
println row['B'] + " "+row['F']+" "+row['G']
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findPeGteCity(cityName, yearmonth){
|
||||
def sql = "SELECT gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
map.deal_pe = r.gujiamdltst_ppe15_pct
|
||||
map.third_pe = r.gujiatrdpttstppe15pct
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-27")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2023-05-27", "2023-06-26")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
|
||||
+10
@@ -17,4 +17,14 @@ class PgsqlCommUtils {
|
||||
});
|
||||
return set
|
||||
}
|
||||
|
||||
|
||||
public static def findAllDistrictCount(def cityid){
|
||||
def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
+83
-36
@@ -33,8 +33,8 @@ import java.util.regex.*;
|
||||
def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
//def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
@@ -45,8 +45,8 @@ DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_craw
|
||||
def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
//def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
// "postgres", "postgres", "org.postgresql.Driver")
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
//def citys = ["山南","黄南州","甘南","克孜勒苏","嘉峪关","海北州","阿拉善盟","怒江州","和田","日喀则","昌都","海南州","玉树州","吐鲁番","阿勒泰","三沙","塔城","甘孜州","果洛州","林芝","那曲","阿坝州","神农架"]
|
||||
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
@@ -1287,50 +1287,50 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\人工指导价录入_1020.xlsx"
|
||||
//
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
|
||||
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['A']){
|
||||
//// def region = RegionData.formatRegion(d['A'], d['B'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['C'], "district")
|
||||
//// if(dist){
|
||||
//// d['E'] = dist.region
|
||||
//// d['F'] = dist._id.toString()
|
||||
//// }
|
||||
//// }
|
||||
// if(!d['F']){
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
////
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
// if(dist){
|
||||
//// d['H'] = dist.region
|
||||
//// d['G'] = dist.name
|
||||
// d['F'] = dist.status
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
//// if(!d['F']){
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//////
|
||||
//// if(dist){
|
||||
////// d['H'] = dist.region
|
||||
////// d['G'] = dist.name
|
||||
//// d['F'] = dist.status
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
|
||||
def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
if(count && count==1){
|
||||
def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
|
||||
d['E'] = dist.city
|
||||
}
|
||||
|
||||
// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dist){
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['J'] = dist.status
|
||||
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
|
||||
// if(count && count==1){
|
||||
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
|
||||
// d['E'] = dist.city
|
||||
// }
|
||||
// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
// d['F'] = newRegion
|
||||
},0)
|
||||
//
|
||||
//// def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['J'] = dist.status
|
||||
//// }
|
||||
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
|
||||
//// d['F'] = newRegion
|
||||
//},0)
|
||||
|
||||
//ExcelUtils.modify(fileName, {d->
|
||||
// if(d['A']){
|
||||
@@ -1364,6 +1364,53 @@ def findGuidedPrice(sqlInstance, id){
|
||||
return count
|
||||
}
|
||||
|
||||
def findPgDealPrice(c, pgsqlInstance, startDate){
|
||||
def sql = "select count(a.gujia_comm_id) from (SELECT distinct gujia_comm_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${c}' and txn_dt>='${startDate}') a"
|
||||
def count = 0
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
citys = CcbCommonUtils.citys_all
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
|
||||
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
|
||||
def rows = []
|
||||
def headers = []
|
||||
headers << "城市"
|
||||
headers.addAll(dates)
|
||||
rows << headers
|
||||
for(def c : citys){
|
||||
def row = []
|
||||
row << c
|
||||
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
|
||||
// for(def d : dates){
|
||||
def startDate = "2022-07-01"
|
||||
def count = findPgDealPrice(c, pgsqlInstance, startDate)
|
||||
row << count*100/disCount
|
||||
|
||||
// def map = findSampleRate(pgsqlInstance, c, d)
|
||||
// row << map.third
|
||||
// }
|
||||
rows << row
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findSampleRate(pgsqlInstance, cityName, yearmonth){
|
||||
def sql = "SELECT urbn_nm,gujiamdltst_ppe15_pct,gujiatrdpttstppe15pct from t_app_p9pha_plda.avm_evaluation_gj_h where urbn_nm='${cityName}' and gujia_bds_yrmo=${yearmonth}"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
map.deal = r.gujiamdltst_ppe15_pct
|
||||
map.third = r.gujiatrdpttstppe15pct
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+1
-1
@@ -87,7 +87,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true);
|
||||
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
|
||||
|
||||
saveToPgSql(rows)
|
||||
|
||||
|
||||
+6
-5
@@ -45,6 +45,7 @@ class TradeStatLog {
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
|
||||
// task.deleteDistrict()
|
||||
// task.deleteDealData()
|
||||
// task.deleteDealData2()
|
||||
@@ -92,13 +93,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys_all;
|
||||
def rets = []
|
||||
rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","23年3月总量","23年4月总量","23年5月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2023-02-01"
|
||||
def date = "2023-03-01"
|
||||
def endDate = "2023-04-01"
|
||||
def newDate = "2023-05-01"
|
||||
def startDate = "2023-03-01"
|
||||
def date = "2023-04-01"
|
||||
def endDate = "2023-05-01"
|
||||
def newDate = "2023-06-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+60
-17
@@ -19,9 +19,9 @@ class CcbDataStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new CcbDataStat()
|
||||
// task.run()
|
||||
task.run(args)
|
||||
// task.tradeRun()
|
||||
task.regionDistStat()
|
||||
// task.regionDistStat()
|
||||
}
|
||||
|
||||
def regionDistStat(){
|
||||
@@ -124,22 +124,37 @@ class CcbDataStat {
|
||||
return count
|
||||
}
|
||||
|
||||
def run(){
|
||||
def run(args){
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\92个城市字段覆盖率情况.xlsx",{ d->
|
||||
if(d['A']) {
|
||||
def citycode = cityCodeDataMap.get(d['A'])
|
||||
def total = findTotalCount(citycode)
|
||||
def f_count = findDistFinishDate(citycode)
|
||||
def b_count = findBuildTypeCount(citycode)
|
||||
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
|
||||
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
|
||||
d['C'] = f_pre
|
||||
d['D'] = b_pre
|
||||
d['E'] = total
|
||||
println d['A'] +" "+ total +" "+f_count+" "+b_count
|
||||
}
|
||||
})
|
||||
def rets = []
|
||||
rets << ["城市","经纬度","物业类型","建筑年代","建筑类型","绿化率","容积率"]
|
||||
for(def c : citys){
|
||||
def ret = []
|
||||
ret << c
|
||||
def citycode = cityCodeDataMap.get(c)
|
||||
def total = findTotalCount(citycode)
|
||||
def lat_count = findLocationCount(citycode)
|
||||
def f_count = findDistFinishDate(citycode)
|
||||
def b_count = findBuildTypeCount(citycode)
|
||||
def g_count = findVirescencePercent(citycode)
|
||||
def c_count = findCubagePercentPercent(citycode)
|
||||
def location_pre = DataHandleUtils.getDoubleRound(2, lat_count*100/total)
|
||||
def f_pre = DataHandleUtils.getDoubleRound(2, f_count*100/total)
|
||||
def b_pre = DataHandleUtils.getDoubleRound(2, b_count*100/total)
|
||||
def g_pre = DataHandleUtils.getDoubleRound(2, g_count*100/total)
|
||||
def c_pre = DataHandleUtils.getDoubleRound(2, c_count*100/total)
|
||||
ret << location_pre
|
||||
ret << 100
|
||||
ret << b_pre
|
||||
ret << f_pre
|
||||
ret << g_pre
|
||||
ret << c_pre
|
||||
ret << total
|
||||
rets << ret
|
||||
}
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\建行供数重要字段覆盖率.xlsx", rets, "统计")
|
||||
}
|
||||
|
||||
def findTotalCount(citycode){
|
||||
@@ -168,4 +183,32 @@ class CcbDataStat {
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findLocationCount(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and longitude>0";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
def findVirescencePercent(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and GREENINGRATE is not null";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
def findCubagePercentPercent(citycode){
|
||||
def sql = "select count(*) as count from bigdata_community_gj where cityid='${citycode}' and status=0 and PLOTRATIO is not null";//where cityId in (350200,150100)
|
||||
def count = 0
|
||||
sqlInstance.eachRow(sql, { r ->
|
||||
count = r.count
|
||||
})
|
||||
return count
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -61,9 +61,9 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
|
||||
|
||||
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
|
||||
cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
// districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
// cityRun(citys, startDate, endDate, cityCodeMap)
|
||||
// regionRun(citys, startDate, endDate, cityCodeMap)
|
||||
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
|
||||
}
|
||||
|
||||
def cityRun(citys, startDate, endDate, cityCodeDataMap){
|
||||
|
||||
+6
-6
@@ -1184,15 +1184,15 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
|
||||
println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
|
||||
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
|
||||
println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
@@ -1273,9 +1273,9 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
def headers = []
|
||||
def allRows = []
|
||||
|
||||
+77
-2
@@ -40,19 +40,26 @@ class HailanTest {
|
||||
static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new HailanTest();
|
||||
task.run();
|
||||
}
|
||||
|
||||
def run(){
|
||||
def inputfile = "D:\\数据\\微元素\\排查表.xlsx"
|
||||
match(inputfile)
|
||||
def inputfile = "D:\\数据\\微元素\\排查表-价格对比-新.xlsx"
|
||||
// addThirdPrice(inputfile)
|
||||
// match(inputfile)
|
||||
// tradeAvgPrice(inputfile)
|
||||
// caluateTotalPrice(inputfile)
|
||||
// addPrice(inputfile)
|
||||
// addCommId("D:\\数据\\海南\\匹配上的合并数据.xlsx")
|
||||
// addHouse(inputfile)
|
||||
// stat(inputfile)
|
||||
// hailanStat(inputfile)
|
||||
// addFzFjwPrice(inputfile)
|
||||
addFjwTobPrice(inputfile)
|
||||
}
|
||||
|
||||
def addHouse(inputfile){
|
||||
@@ -702,4 +709,72 @@ class HailanTest {
|
||||
}
|
||||
return prices;
|
||||
}
|
||||
|
||||
def tradeAvgPrice(inputfile){
|
||||
ExcelUtils.modify(inputfile, { row ->
|
||||
if(row['I'] && row['I'] == "4f4dac0a84ae1852a920ad39"){
|
||||
def dist = CommonDatautils.findDistrictById(row['I'])
|
||||
if(dist){
|
||||
def tradeAvgPrice = CommonDatautils.findTrades(dist, "2023-02-01", "2023-06-01");
|
||||
row['J'] = (int)tradeAvgPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addThirdPrice(path){
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
|
||||
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
def cityMap = CommonDatautils.findThirdPartyPrice("福州", row['F'], row['G'], startDate, endDate)
|
||||
def priceMap = cityMap.get("福州"+"##"+row['F']+"##"+row['G'])
|
||||
if(priceMap){
|
||||
if(priceMap.containsKey("lianjia_esf_price")){
|
||||
row['K'] = (int)(priceMap.get("lianjia_esf_price")*0.95)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def caluateTotalPrice(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(row['J']){
|
||||
row['L'] = (int)(row['J'].toString().toDouble()*row['B'].toString().toDouble())
|
||||
}else if(row['K']){
|
||||
row['L'] = (int)(row['K'].toString().toDouble()*row['B'].toString().toDouble())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def addFjwTobPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(d['F']){
|
||||
def dbo = CommonDatautils.queryDistrictPrice("福州", d['F'], d['G'], DateUtils.parseFromTightByPattern("2023-07-02","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
d['Q'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addFzFjwPrice(inputfile){
|
||||
ExcelUtils.modify(inputfile, { row ->
|
||||
if(row['B']) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", "福州")
|
||||
q.put("region", row['A'])
|
||||
q.put("name", row['B'])
|
||||
q.put("month", 202304)
|
||||
def obj = FZ_DISTRICT_STAT_COLL.findOne(q)
|
||||
if (obj) {
|
||||
row['D'] = obj.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+119
-4
@@ -1,7 +1,9 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
@@ -250,7 +252,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "³¤Ì²ÃÀÊûÈýÆÚ")
|
||||
// q.put("name", "鲁能大厦")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -737,8 +739,9 @@ class CommonDatautils {
|
||||
public static def findAllDistrict(def cat, def fields){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("visible",0)
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
// q.put("name", "中兴智汇大厦")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -746,7 +749,8 @@ class CommonDatautils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
rets << d
|
||||
@@ -1022,7 +1026,7 @@ class CommonDatautils {
|
||||
q.put("cat", cat);
|
||||
q.put("region", region)
|
||||
q.put("name", name);
|
||||
q.put("status", 0)
|
||||
q.put("visible", 0)
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -3801,5 +3805,116 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.city+"##"+db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.beike_esf_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 根据经纬度获取区域
|
||||
* @param city
|
||||
* @param location
|
||||
* @return
|
||||
*/
|
||||
static def getRegionByLocation(city, def location){
|
||||
def json = GaoDeApiUtils.geocode(location);
|
||||
def region = null;
|
||||
if (json && json.info=="OK") {
|
||||
def regeocode = json.regeocode
|
||||
if(regeocode && regeocode.addressComponent){
|
||||
region = regeocode.addressComponent.district
|
||||
if(!region){
|
||||
region = regeocode.addressComponent.township
|
||||
}
|
||||
}
|
||||
}
|
||||
if(region){
|
||||
region = RegionData.formatRegion(city, region)
|
||||
}
|
||||
return region;
|
||||
}
|
||||
|
||||
def static getAllFiles(def path) {
|
||||
|
||||
File file = new File(path);
|
||||
File[] tempList = file.listFiles();
|
||||
|
||||
def files = [];
|
||||
for (int i = 0; i < tempList.length; i++) {
|
||||
if (tempList[i].isFile()) {
|
||||
files << tempList[i];
|
||||
}
|
||||
if (tempList[i].isDirectory()) {
|
||||
System.out.println("文件夹:" + tempList[i]);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+18
-3
@@ -250,8 +250,23 @@ class DataHandleUtils {
|
||||
return sum;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public static isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街道|大道|巷|胡同|街|坊]\\d+(-\\d+)?(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static main(args) {
|
||||
def a = isStandardRoad("杨桥西路236号")
|
||||
println a
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package com.fangjia.script.district
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
//删除别名是另一个小区的主名
|
||||
class MasterNameInAlias {
|
||||
|
||||
static main(args) {
|
||||
def task = new MasterNameInAlias()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
def allList = []
|
||||
for(def city : citys){
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","address","alias","estate_type2", "lat", "lng"])
|
||||
|
||||
def cityList = []
|
||||
cityList << ["城市","区域","名称","物业类型","地址","别名", "距离", "地址状态"]
|
||||
for(def dist1 : disticts){
|
||||
def distList = []
|
||||
for(def dist2 : disticts){
|
||||
if(dist1._id && dist2._id && dist1._id.toString()!=dist2._id.toString()){
|
||||
if(dist2.alias && dist2.alias.contains(dist1.name)){
|
||||
def distance = ''
|
||||
if(dist1.lng && dist2.lng){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, dist2.lng/10000000000d, dist2.lat/10000000000d);
|
||||
}
|
||||
distList << [dist2.city, dist2.region, dist2.name, dist2.estate_type2, dist2.address, dist2.alias?dist2.alias.join(' '):'', distance]
|
||||
dist2.alias.remove(dist1.name)
|
||||
def d = new BasicDBObject()
|
||||
d.put("_id", dist2._id)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", dist2.alias)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(d, new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
if(distList){
|
||||
cityList << [dist1.city, dist1.region, dist1.name, dist1.estate_type2, dist1.address, dist1.alias?dist1.alias.join(' '):'', '']
|
||||
cityList.addAll(distList)
|
||||
// cityList << []
|
||||
}
|
||||
}
|
||||
|
||||
if(cityList){
|
||||
allList.addAll(cityList)
|
||||
println city +" "+cityList.size()
|
||||
}
|
||||
}
|
||||
// ExcelUtils.write("d:\\数据\\小区\\主名在别名中.xlsx", allList, "数据")
|
||||
}
|
||||
}
|
||||
+616
@@ -0,0 +1,616 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.sql.ResultSetMetaData
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class AmapAddressUpdate {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static main(args) {
|
||||
def task = new AmapAddressUpdate()
|
||||
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
|
||||
def addr = task.formatAddress("福州", "北三环路201号")
|
||||
println addr
|
||||
// task.run(path)
|
||||
// task.similarMatch(path)
|
||||
// task.addressRun(path)
|
||||
// task.addressSource(path)
|
||||
// task.amapRun()
|
||||
|
||||
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
|
||||
// task.updateData(path)
|
||||
// task.updateLocation(path)//更新经纬度
|
||||
}
|
||||
def run(path){
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity()
|
||||
def allrets = []
|
||||
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
|
||||
for (def city : citys) {
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
|
||||
def rets = findAmapDetail(disticts)
|
||||
allrets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write(path, allrets, "数据")
|
||||
}
|
||||
|
||||
def amapRun(){
|
||||
def city = "福州市"
|
||||
def datas = findAllAmapByCity(city)
|
||||
// datas = datas[0..10]
|
||||
println "Es总量:"+datas.size()
|
||||
def rets = match(datas)
|
||||
|
||||
ExcelUtils.write("d:\\数据\\小区\\福州高德数据匹配.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
def match(datas){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
rets << ["ID","高德城市","高德区域","高德名称","高德地址","匹配城市","匹配区域","匹配名称","匹配类型","匹配ID"]
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = []
|
||||
def jsonObject = OpenApiClient.addressMatch(d.city, d.name, "property", false, d.region, null)
|
||||
ret << d.id
|
||||
ret << d.city
|
||||
ret << d.region
|
||||
ret << d.name
|
||||
ret << d.address
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
ret << jsonObject.get("city")
|
||||
ret << jsonObject.get("district")
|
||||
ret << jsonObject.get("name")
|
||||
ret << jsonObject.get("propertyType")
|
||||
ret << jsonObject.get("id")
|
||||
}
|
||||
println totalCount.get() +' '+ret
|
||||
rets << ret
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def findAllAmapByCity(city){
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT id,cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and type like '%住宅%'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def list = []
|
||||
while (rs.next()) {
|
||||
def map = [:]
|
||||
map.id = rs.getString("id")
|
||||
map.city = city
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.region = rs.getString("adname")
|
||||
// map.type = rs.getString("type")
|
||||
// def location = rs.getObject("location")
|
||||
// if(location){
|
||||
// map.lng = location.x
|
||||
// map.lat = location.y
|
||||
// }
|
||||
list << map
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findAmapDetail(districts){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def d = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def row = []
|
||||
row << d.city
|
||||
row << d.region
|
||||
row << d.name
|
||||
row << d.address
|
||||
if(d.addr_alias){
|
||||
row << d.addr_alias.join(" ")
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
def dbo = getAmapLocation(d.city+"市", d.region, d.name)
|
||||
if(!dbo){
|
||||
def newName = d.name
|
||||
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
|
||||
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
|
||||
}
|
||||
dbo = getAmapLocation(d.city+"市", d.region, newName)
|
||||
}
|
||||
|
||||
// if(!dbo && d.lat){
|
||||
// dbo = getAmapLocation(d.city+"市", d.region, newName, d.lat/10000000000.0, d.lng/10000000000.0)
|
||||
// }
|
||||
if(dbo){
|
||||
row << dbo.region
|
||||
row << dbo.name
|
||||
row << dbo.address
|
||||
row << formatAddress(d.city, dbo.address)
|
||||
row << dbo.lng
|
||||
row << dbo.lat
|
||||
row << dbo.match_type
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
}
|
||||
println totalCount.get() +' '+row
|
||||
rets << row
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def formatAddress(def city, def address){
|
||||
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
|
||||
def address_str = null
|
||||
if(jsonObj && jsonObj.result) {
|
||||
def street = ''
|
||||
def street_number = ''
|
||||
if (jsonObj.result.street) {
|
||||
street = jsonObj.result.street
|
||||
}
|
||||
if (jsonObj.result.streetNumber) {
|
||||
street_number = jsonObj.result.streetNumber
|
||||
}
|
||||
if (street || street_number) {
|
||||
address_str = street
|
||||
if(street_number){
|
||||
address_str = address_str + street_number
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(address_str && address_str.contains("交叉口")){
|
||||
address_str = address_str.substring(0, address_str.indexOf("交叉口")+3)
|
||||
}
|
||||
return address_str;
|
||||
}
|
||||
|
||||
def getAmapLocation(city, region, name){
|
||||
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
sqlBuffer.append(" and type like '%住宅%'")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def map = [:]
|
||||
while (rs.next()) {
|
||||
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
|
||||
def mapName = rs.getString("name").replace("·","")
|
||||
if(mapName == name && rs.getString("adname").contains(region)){
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
map.formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
map.region = rs.getString("adname")
|
||||
map.type = rs.getString("type")
|
||||
map.match_type = "名称相同"
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// String sql = "select * from amap_poi_detail where cityname='${city}' and name='${name}' and type like '%住宅%'"
|
||||
// def loc = null
|
||||
// ElasticUtils.executeSqlQuery(sql,{ data->
|
||||
// println data.adname+" "+data.name +" "+data.address +" "+data.type
|
||||
// if(name == data.name && data.adname.contains(region) && data.type.contains("type")){
|
||||
// data.formatAddress = AddressUtil.formatAddress(city, data.address)
|
||||
// loc = data
|
||||
// }
|
||||
// })
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def addressRun(inputfile){
|
||||
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
// if(row['H'] && row['H'].contains("口步行") && row['H'].contains("(")){
|
||||
// row['I'] = row['H'].substring(0, row['H'].indexOf("("))
|
||||
// }
|
||||
if(row['J']){
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
def distance = null
|
||||
if(dist1){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, row['J'].toString().toDouble(), row['K'].toString().toDouble());
|
||||
}
|
||||
row['M'] = distance
|
||||
}
|
||||
|
||||
def gaodeRoadFlag = false
|
||||
if(row['I']){
|
||||
gaodeRoadFlag = DataHandleUtils.isStandardRoad(row['I'])
|
||||
}
|
||||
def dbRoadFlag = false
|
||||
if(row['D']){
|
||||
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
|
||||
}
|
||||
if(!row['N'] && row['D'] && row['I']){
|
||||
if(row['D']==row['I']){
|
||||
row['N'] = 100
|
||||
}else if(row['E'] && gaodeRoadFlag && row['E'].contains(row['I'])){
|
||||
row['N'] = 1
|
||||
}
|
||||
}
|
||||
|
||||
if(!row['N'] && !dbRoadFlag && gaodeRoadFlag){
|
||||
row['N'] = 2
|
||||
}
|
||||
|
||||
if(!row['N'] && dbRoadFlag && gaodeRoadFlag && row['I'].contains(row['D'])){
|
||||
row['N'] = 3
|
||||
}
|
||||
|
||||
if(!row['N'] && dbRoadFlag && gaodeRoadFlag){
|
||||
row['N'] = 4
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def addressSource(inputfile) {
|
||||
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(!row['N'] && row['D'] && !row['O']){
|
||||
def addresses = findAllEsData(row['A'], row['B'], row['C'])
|
||||
if(addresses){
|
||||
def dbRoadFlag = false
|
||||
if(row['D']){
|
||||
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
|
||||
}
|
||||
if(dbRoadFlag){
|
||||
def list = []
|
||||
list << row['D']
|
||||
if(row['E']){
|
||||
def arr = row['E'].split(" ")
|
||||
list.addAll(arr as List)
|
||||
}
|
||||
for(def addr : list){
|
||||
if(addresses.contains(addr)){
|
||||
row['O'] = "诸葛找房"
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def findAllEsData(city,region, name){
|
||||
def rets = []
|
||||
def sql = "select * from community where city='${city}' and name='${name}' and source='诸葛找房APP'"
|
||||
rets = findDatas(region, sql)
|
||||
if(!rets){
|
||||
rets = findAliasDatas(city,region, name)
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findDatas(region, sql){
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
if(d.region && d.region.contains(region) && d.address){
|
||||
rets << d.address
|
||||
}
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findAliasDatas(city, region, name){
|
||||
def must = []
|
||||
def term1 = ["match":["city":city]]
|
||||
must << term1
|
||||
|
||||
def should = []
|
||||
should << ["match":["alias":name]]
|
||||
def params = ["query":["bool":["must":must, "should":should]]]
|
||||
def s = EsUtils.httpSaveData(params, "http://192.168.0.206:9200/community/_doc/_search");
|
||||
def rets = [];
|
||||
if(s) {
|
||||
def doc = new groovy.json.JsonSlurper().parseText(s)
|
||||
def hits = doc.hits
|
||||
if(hits){
|
||||
hits = hits.hits
|
||||
for(def h : hits){
|
||||
def d = h._source
|
||||
if(d.region && d.region.contains(region) && d.address && d.source=="诸葛找房APP"){
|
||||
rets << d.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def addrAndLocationDistance(inputfile){
|
||||
ExcelUtils.multiThreadingModify(inputfile, { row ->
|
||||
if(!row['N'] && row['D']){
|
||||
println row['D']
|
||||
def dbRoadFlag = false
|
||||
if(row['D']){
|
||||
dbRoadFlag = DataHandleUtils.isStandardRoad(row['D'])
|
||||
}
|
||||
if(dbRoadFlag){
|
||||
def gloJson = GaoDeApiUtils.geo(row['A'], row['D'])
|
||||
if(gloJson) {
|
||||
def glocation = null
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def map = geocodes.get(0)
|
||||
if(map.formatted_address.contains(row['D'])){
|
||||
def arrs = map.location.split(",")
|
||||
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
def distance = null
|
||||
if(dist1){
|
||||
distance = com.fangjia.core.util.GeoUtils.calculateDistance(dist1.lng/10000000000d, dist1.lat/10000000000d, arrs[0].toString().toDouble(), arrs[1].toString().toDouble());
|
||||
}
|
||||
row['P'] = distance
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateData(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(row['Q'] && row['Q'].contains("修改地址")){
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
def udata = new BasicDBObject()
|
||||
if(row['Q'].contains("可替换原地址")){
|
||||
udata.put("address", row['I'])
|
||||
}else if(row['Q'].contains("可新增地址别名")){
|
||||
udata.put("address", row['I'])
|
||||
def addrAlias = dist1.addr_alias
|
||||
if(!addrAlias){
|
||||
addrAlias = []
|
||||
}
|
||||
addrAlias << dist1.address
|
||||
udata.put("addr_alias", addrAlias)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def updateLocation(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
|
||||
if(row['R'] && row['R']=="是"){
|
||||
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
if(dist1){
|
||||
def udata = new BasicDBObject()
|
||||
def lat = null
|
||||
def lng = null
|
||||
if(row['J']){
|
||||
lng = row['J'].toString().toDouble()
|
||||
lat = row['K'].toString().toDouble()
|
||||
}else{
|
||||
def gloJson = GaoDeApiUtils.geo(row['A'], row['D'])
|
||||
if(gloJson) {
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def map = geocodes.get(0)
|
||||
if (map.formatted_address.contains(row['D'])) {
|
||||
def arrs = map.location.split(",")
|
||||
lng = arrs[0].toString().toDouble()
|
||||
lat = arrs[1].toString().toDouble()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 高德转百度
|
||||
def baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
def lng2 = baidu.x
|
||||
def lat2 = baidu.y
|
||||
Long lng_new = lng *10000000000L
|
||||
Long lat_new = lat *10000000000L
|
||||
udata.put("lng", lng_new)
|
||||
udata.put("lat", lat_new)
|
||||
if(lng2){
|
||||
Long lng2_new = lng2 *10000000000L
|
||||
Long lat2_new = lat2 *10000000000L
|
||||
udata.put("lng2", lng2_new)
|
||||
udata.put("lat2", lat2_new)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist1.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def similarMatch(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['F']){
|
||||
def d = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
|
||||
if(d && d.lat){
|
||||
def dbo = getSimilarAmapLocation(d.city+"市", d.region, d.name, d.address, d.lat/10000000000.0, d.lng/10000000000.0)
|
||||
if(dbo){
|
||||
row['F'] = dbo.region
|
||||
row['G'] = dbo.name
|
||||
row['H'] = dbo.address
|
||||
row['I'] = formatAddress(d.city, dbo.address)
|
||||
row['J'] = dbo.lng
|
||||
row['K'] = dbo.lat
|
||||
row['L'] = dbo.score
|
||||
row['M'] = dbo.distance
|
||||
row['S'] = dbo.type
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getSimilarAmapLocation(city, region, name, address, lat, lng){
|
||||
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
|
||||
sqlBuffer.append(" and cityname = '" + city + "'")
|
||||
sqlBuffer.append(" and MATCH(name,'${name}')")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
|
||||
def maxSocreObj = null
|
||||
while (rs.next()) {
|
||||
def map = [:]
|
||||
def mapName = rs.getString("name").replace("·","")
|
||||
|
||||
println name +" "+ mapName
|
||||
if(rs.getString("adname").contains(region)){
|
||||
map.name = rs.getString("name")
|
||||
map.address = rs.getString("address")
|
||||
|
||||
map.region = rs.getString("adname")
|
||||
map.type = rs.getString("type")
|
||||
map.match_type = "名称相似"
|
||||
def location = rs.getObject("location")
|
||||
if(location){
|
||||
map.lng = location.x
|
||||
map.lat = location.y
|
||||
}
|
||||
|
||||
double nameScore = 0
|
||||
double addScore = 0
|
||||
double loctionScore = 0
|
||||
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(lng, lat, map.lng.toString().toDouble(), map.lat.toString().toDouble());
|
||||
map.distance = distance
|
||||
if(map.name && name){
|
||||
nameScore = CosineSimilarAlgorithm.getSimilarity(map.name, name);
|
||||
}
|
||||
if(map.address && address){
|
||||
addScore = CosineSimilarAlgorithm.getSimilarity(map.address, address);
|
||||
}
|
||||
|
||||
if(hasColumn(rs, "shape")){
|
||||
def shape = rs.getObject("shape")
|
||||
if(shape){
|
||||
def coordinates = shape.getObject("coordinates")
|
||||
// def pointList = InsidePolygon.createPoint2(points);
|
||||
def flag = InsidePolygon.isInArea(lng, lat, coordinates)
|
||||
if(flag){
|
||||
loctionScore = 0.5
|
||||
}
|
||||
}
|
||||
}else{
|
||||
|
||||
if(distance < 500 && (nameScore>0.5 || addScore>0.5)){
|
||||
loctionScore = 0.3
|
||||
}
|
||||
}
|
||||
|
||||
def score = nameScore*0.3 + addScore*0.2 + loctionScore
|
||||
map.score = score
|
||||
if(!maxSocreObj){
|
||||
if(score>0){
|
||||
map.formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
maxSocreObj = map
|
||||
}
|
||||
}else{
|
||||
println score +" "+maxSocreObj.score
|
||||
if(maxSocreObj && score>maxSocreObj.score){
|
||||
map.formatAddress = AddressUtil.formatAddress(city, map.address)
|
||||
maxSocreObj = map
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return maxSocreObj;
|
||||
}
|
||||
|
||||
|
||||
def hasColumn(ResultSet rs, String columnName) {
|
||||
ResultSetMetaData rsmd = rs.getMetaData();
|
||||
int columns = rsmd.getColumnCount();
|
||||
for (int x = 1; x <= columns; x++) {
|
||||
if (columnName.equals(rsmd.getColumnName(x))) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+2
@@ -10,6 +10,7 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -321,6 +322,7 @@ class DistrictInfoExportBatch extends BaseBatch{
|
||||
def s = str.toString();
|
||||
index++;
|
||||
if(s && s!="null"){
|
||||
s = s.replace('"','\\"')
|
||||
// if(s.toString().contains(",")){
|
||||
// s = csvHandlerStr(s.toString())
|
||||
//
|
||||
|
||||
+16
-12
@@ -60,7 +60,8 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def calcMonthlyPrice(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
//Ö»Óи£ÖÝʹÓÃtob¼Û¸ñ
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def keys = new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("estate_type2", 1)
|
||||
|
||||
def lastMonthWeekend = DateUtils.truncateDay(DateUtils.getLastSundayInWeekOfLastMonth(new Date()))
|
||||
@@ -142,17 +143,15 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
|
||||
if(weekendPriceObj.value_type!=1){
|
||||
if(dist.city == "¸ŁÖÝ"){
|
||||
if(Math.abs(mom)*100>3){
|
||||
if(dist.region != "Âíβ"){
|
||||
def randMarkup = nextDouble(0, 3)
|
||||
if(mom < 0){
|
||||
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
}else{
|
||||
mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
}
|
||||
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
}
|
||||
}
|
||||
// if(Math.abs(mom)*100>3){
|
||||
// def randMarkup = nextDouble(0, 3)
|
||||
// if(mom < 0){
|
||||
// mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
|
||||
// }else{
|
||||
// mom = DataUtil.roundDouble(randMarkup/100, 4)
|
||||
// }
|
||||
// curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
|
||||
// }
|
||||
}else{
|
||||
if(Math.abs(mom)*100>10){
|
||||
def randMarkup = nextDouble(0, 10)
|
||||
@@ -232,6 +231,11 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def dbcursor = DISTRICT_STAT.find(q)
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(city == "¸£ÖÝ"){
|
||||
if(d.avg_price_2b){
|
||||
d.avg_price = d.avg_price_2b
|
||||
}
|
||||
}
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
}
|
||||
return priceMap
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class FzDataResultStat {
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
task.priceMarkupRun();
|
||||
// task.regionPriceTrendRun()
|
||||
task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
}
|
||||
|
||||
+4
-1
@@ -114,7 +114,10 @@ class HouseExportBatch extends BaseBatch{
|
||||
// sb.append(s+ "@|@")
|
||||
// }
|
||||
// }
|
||||
sb.append(StringUtils.join(row.toArray(), "@|@"))
|
||||
def str = StringUtils.join(row.toArray(), "@|@")
|
||||
str = str.replace('"','\\"')
|
||||
sb.append(str)
|
||||
|
||||
sb.append(System.getProperty("line.separator"));
|
||||
|
||||
println index
|
||||
|
||||
@@ -2,6 +2,9 @@ package com.fangjia.script.fuzhou
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.JsonUtils
|
||||
import com.fangjia.core.util.Message
|
||||
import com.fangjia.core.util.MessageInfo
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -9,26 +12,71 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
|
||||
def city = RegionData.formatCity("余姚")
|
||||
d['D'] = city
|
||||
if(city){
|
||||
d['E'] = RegionData.formatRegion(city, d['C'])
|
||||
}
|
||||
|
||||
// if(!d['D']){
|
||||
// def citys = CommonDatautils.getCityByRegion(d['A'])
|
||||
// if(citys && citys.size()==1){
|
||||
// d['F'] = citys.get(0)
|
||||
// }
|
||||
//Message message = new Message();
|
||||
//message.setMsgtype("text");
|
||||
//message.setText(new MessageInfo("测试"));
|
||||
//URL url = new URL("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430");
|
||||
//// 建立http连接
|
||||
//HttpURLConnection conn = (HttpURLConnection) url.openConnection();
|
||||
//conn.setDoOutput(true);
|
||||
//conn.setDoInput(true);
|
||||
//conn.setUseCaches(false);
|
||||
//conn.setRequestMethod("POST");
|
||||
//conn.setRequestProperty("Charset", "UTF-8");
|
||||
//conn.setRequestProperty("Content-Type", "application/Json; charset=UTF-8");
|
||||
//conn.connect();
|
||||
//OutputStream out = conn.getOutputStream();
|
||||
//String textMessage = JsonUtils.toJson(message);
|
||||
//byte[] data = textMessage.getBytes("UTF-8");
|
||||
//out.write(data);
|
||||
//out.flush();
|
||||
//out.close();
|
||||
//System.out.println(conn.getResponseCode());
|
||||
//InputStream ins = conn.getInputStream();
|
||||
//byte[] data1 = new byte[ins.available()];
|
||||
//ins.read(data1);
|
||||
//System.out.println(new String(data1));
|
||||
|
||||
//HttpRequest request = new HttpRequest()
|
||||
//def params = [:]
|
||||
//params.put("","")
|
||||
//request.post("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430","", "utf-8")
|
||||
|
||||
|
||||
//sendMarkdownMessage("租金", "测试")
|
||||
//static void sendMarkdownMessage(String title, String msg){
|
||||
// HttpRequest request = new HttpRequest()
|
||||
// JSONObject requestBody = new JSONObject()
|
||||
// JSONObject markdown = new JSONObject()
|
||||
// markdown.put("title", title)
|
||||
// markdown.put("text", msg)
|
||||
// requestBody.put("markdown", markdown)
|
||||
// requestBody.put("msgtype", "markdown")
|
||||
// request.postWithBody("https://oapi.dingtalk.com/robot/send?access_token=35587c9e7accc54ff7ddf2c6ebdc36a3ba065b4a2bdac78c0e9928596361b430", requestBody, null, "UTF-8")
|
||||
//}
|
||||
//ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\第三方小区区域.xlsx", { d ->
|
||||
// def city = RegionData.formatCity("余姚")
|
||||
// d['D'] = city
|
||||
// if(city){
|
||||
// d['E'] = RegionData.formatRegion(city, d['C'])
|
||||
// }
|
||||
})
|
||||
//
|
||||
//// if(!d['D']){
|
||||
//// def citys = CommonDatautils.getCityByRegion(d['A'])
|
||||
//// if(citys && citys.size()==1){
|
||||
//// d['F'] = citys.get(0)
|
||||
//// }
|
||||
//// }
|
||||
//})
|
||||
|
||||
//def datas = ExcelUtils.read("D:\\数据\\福州\\202305月福州统计数据\\福州马尾第三方.xlsx", 0, 1)
|
||||
//def m = [:]
|
||||
@@ -188,6 +236,24 @@ def findThirdPrice(city, region, name, AVERAGE_STATISTICS){
|
||||
//},0)
|
||||
|
||||
|
||||
def files = CommonDatautils.getAllFiles("D:\\数据\\福州\\原始数据")
|
||||
if(files){
|
||||
for(def filepath : files){
|
||||
def rets = read(filepath);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def read(path){
|
||||
File file = new File(path);
|
||||
def rets = []
|
||||
file.eachLine{it->
|
||||
rets << it
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202304.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202305.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
+3
-2
@@ -29,10 +29,11 @@ class AddAvgpriceBatch {
|
||||
}
|
||||
|
||||
def test(){
|
||||
def districts = CommonDatautils.findDistrictByCityAndRegions("¸£ÖÝ","district", ["Âíβ"], ["city","region","name", "estate_type2"])
|
||||
def districts = CommonDatautils.findAllDistricts("¸£ÖÝ","district", ["city","region","name", "estate_type2"])
|
||||
// def districts = CommonDatautils.findDistrictByCityAndRegions("¸£ÖÝ","district", ["Âíβ"], ["city","region","name", "estate_type2"])
|
||||
for(def dist : districts){
|
||||
//发重算均价命令
|
||||
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-05-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-05-30','yyyy-MM-dd'))
|
||||
addPrice(dist.city,dist.region,dist.name, dist.estate_type2, DateUtils.parseFromTightByPattern('2023-06-30','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-07-05','yyyy-MM-dd'))
|
||||
}
|
||||
}
|
||||
def run(){
|
||||
|
||||
@@ -8,6 +8,7 @@ import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
@@ -20,8 +21,10 @@ class FiledStrand {
|
||||
def updateIndex = 0;
|
||||
static main(args) {
|
||||
|
||||
def task = new FiledStrand();
|
||||
task.run();
|
||||
|
||||
|
||||
// def task = new FiledStrand();
|
||||
// task.run();
|
||||
// task.updateOpenDate();
|
||||
}
|
||||
|
||||
@@ -265,13 +268,58 @@ class FiledStrand {
|
||||
static def processArea(all_area){
|
||||
if(all_area){
|
||||
def v = getValue(all_area)
|
||||
if(v>=1000 && v<=10000000){
|
||||
if(v>=5000 && v<=10000000){
|
||||
return v.toString().toDouble().toInteger()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processStandardHeight(height){
|
||||
if(height){
|
||||
def v = getValue(height)
|
||||
if(v>=2.4 && v<=10){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processElevators(elevators){
|
||||
if(elevators){
|
||||
def v = getValue(elevators)
|
||||
if(v>0 && v<=100){
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processStandardArea(area){
|
||||
if(area){
|
||||
area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","")
|
||||
area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll("");
|
||||
return area
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processCarSiteRent(rent){
|
||||
if(rent){
|
||||
def v = getNumberRange(rent)
|
||||
if(v){
|
||||
if(v=="0"){
|
||||
return null
|
||||
}else{
|
||||
return v.toString()
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static def processName(name){
|
||||
name = name.replace('·','').replace('.','').replace('●','').replace('@','').replace('%','').replace('?','').replace('Ⅰ期','一期')
|
||||
if (name.contains('(') && name.contains(')')){
|
||||
@@ -287,25 +335,25 @@ class FiledStrand {
|
||||
|
||||
|
||||
if (name.find('【')!=-1 && name.find('】')!=-1){
|
||||
def rep_str = name[name.index('【'):name.rfind('】')+1]
|
||||
def rep_str = name.subsring(name.index('【'),name.rfind('】')+1)
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('【')!=-1 && name.find('】')!=-1){
|
||||
def rep_str = name[name.index('【'):name.rfind('】')+1]
|
||||
def rep_str = name.subsring(name.index('【'),name.rfind('】')+1)
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('(')!=-1 && name.find(')')!=-1){
|
||||
def rep_str = name[name.index('('):name.rfind(')')+1]
|
||||
def rep_str = name.subsring(name.index('('),name.rfind(')')+1)
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
|
||||
|
||||
if (name.find('「')!=-1 && name.find('」')!=-1){
|
||||
def rep_str = name[name.index('「'):name.rfind('」')+1]
|
||||
def rep_str = name.subsring(name.index('「'),name.rfind('」')+1)
|
||||
name = name.replace(rep_str, '')
|
||||
}
|
||||
return name
|
||||
@@ -338,7 +386,7 @@ class FiledStrand {
|
||||
if(v){
|
||||
if([40,50,70].contains(v)){
|
||||
try {
|
||||
return v.toString()+"年";
|
||||
return v.toString();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
@@ -378,6 +426,16 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processStandardAddress(def esAddress, def city){
|
||||
if(esAddress){
|
||||
def formatAddress = AddressUtil.formatAddress(city, esAddress)
|
||||
if(formatAddress && isStandardRoad(formatAddress)){
|
||||
return formatAddress
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processBlock(city, region, def block){
|
||||
if(block){
|
||||
if(block.contains("无") || block.toString().length()>10){
|
||||
@@ -464,6 +522,14 @@ class FiledStrand {
|
||||
return false
|
||||
}
|
||||
|
||||
static def isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
static def addBuildAreaToData(def dist, def updateFile){
|
||||
@@ -629,7 +695,7 @@ class FiledStrand {
|
||||
println "estateCharege 转不了 "+estateCharege
|
||||
}
|
||||
}
|
||||
return estateCharege;
|
||||
return null;
|
||||
}
|
||||
|
||||
static def processHouseholdCount(def household_count, def estateType2){
|
||||
@@ -666,6 +732,34 @@ class FiledStrand {
|
||||
return null
|
||||
}
|
||||
|
||||
static def processFloor(floor){
|
||||
def v = getValue(floor.toString());
|
||||
if(v){
|
||||
if(v>=1 && v<=130){
|
||||
try {
|
||||
return v.toString();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processRealUsePercent(percent){
|
||||
def v = getValue(percent.toString());
|
||||
if(v){
|
||||
if(v>=20 && v<=80){
|
||||
try {
|
||||
return v.toString();
|
||||
}catch(Exception e){
|
||||
println e.getMessage()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
static def processFinishBuildingDate(def finishBuildingDate){
|
||||
if(finishBuildingDate){
|
||||
Pattern pat1 = Pattern.compile("\\d{4}[年\\-\\.]\\d{2}[月\\-\\.]\\d{2}");
|
||||
@@ -677,24 +771,37 @@ class FiledStrand {
|
||||
Pattern pat3 = Pattern.compile("\\d{4}");
|
||||
def matcher3 = pat3.matcher(finishBuildingDate);
|
||||
|
||||
def dateStr = null
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group().replace("年","-").replace("月","-")
|
||||
dateStr = matcher1.group().replace("年","-").replace("月","-").replace(".","")
|
||||
}else if(matcher2.find()){
|
||||
return matcher2.group().replace("年","-")
|
||||
dateStr = matcher2.group().replace("年","-").replace(".","-")
|
||||
}else if(matcher3.find()){
|
||||
return matcher3.group()
|
||||
dateStr = matcher3.group().replace(".","-")
|
||||
}
|
||||
if(dateStr){
|
||||
dateStr = dateStr.substring(0, 4)
|
||||
}
|
||||
return dateStr
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
static def processCompany(val){
|
||||
if(val){
|
||||
if(val.length()>20){
|
||||
return null
|
||||
}else{
|
||||
if(val.contains("无")){
|
||||
if(val.contains("暂无")){
|
||||
return null
|
||||
}
|
||||
if(val == "无"){
|
||||
return null
|
||||
}
|
||||
if(val.isNumber()){
|
||||
return null
|
||||
}
|
||||
def fiterChars = [",",".","?","/","数据",'-','?',' ',' ']
|
||||
@@ -771,4 +878,31 @@ class FiledStrand {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static def patternParse(String keyword, String patternStr) {
|
||||
Pattern pattern = Pattern.compile(patternStr);
|
||||
Matcher matcher = pattern.matcher(keyword);
|
||||
String num = "";
|
||||
if (matcher.find()) {
|
||||
if (StringUtils.isNotBlank(matcher.group(0))) {
|
||||
num = matcher.group(1).trim();
|
||||
}
|
||||
}
|
||||
return num;
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def getNumberRange(def input) {
|
||||
Pattern pattern = Pattern.compile("\\d+(-\\d+)?");
|
||||
Matcher matcher = pattern.matcher(input);
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list.size()>0){
|
||||
return list.get(0);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -48,11 +48,11 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2023-04-01"
|
||||
static def statDate = "2023-05-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.esfParser("D:\\数据\\友达\\2023年6月\\查询1.DBF");//友达二手房存入mongo
|
||||
// task.newhouseToSaveEs();//新房保存到es
|
||||
task.esfhouseToSaveEs();//二手房保存到es
|
||||
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
|
||||
|
||||
+1
-1
@@ -65,7 +65,7 @@ class AddDistrictElevatorBatch extends BaseBatch{
|
||||
}
|
||||
def newHouses = analyElevator(haveHouses, dist.estate_type2)
|
||||
if(newHouses && newHouses.size()>=3 && newHouses.size()>=haveHouses.size()*0.6){
|
||||
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 10层以上:"+newHouses.size()
|
||||
println totalCount.get()+" [${dist.estate_type2}] "+dist.name +" 原:"+houses.size()+" 有值:"+haveHouses.size()+" 筛选样本量:"+newHouses.size()
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
if(dist.estate_type2 == "±ðÊû"){
|
||||
udata.put("is_elevator", false)
|
||||
|
||||
+19
@@ -1,9 +1,14 @@
|
||||
package com.fangjia.script.gj.district
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.statistics.poi.AmapPoi
|
||||
import com.fangjia.script.temp.shenjian.util.GaoDeUtils
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.HttpRequestUtil
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
class FangjiaClientUtils {
|
||||
@@ -153,4 +158,18 @@ class FangjiaClientUtils {
|
||||
return columnValMap;
|
||||
}
|
||||
|
||||
|
||||
public static def saveOrUpdateDistrict(data) {
|
||||
String jsonStr = com.alibaba.fastjson.JSONObject.toJSONString(data)
|
||||
// def json = HttpUtils.postForJson("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
|
||||
// def json = HttpUtils.postForJson("http://localhost:8080/FangJiaDataService/saveData", jsonStr)
|
||||
|
||||
com.alibaba.fastjson.JSONObject json = HttpUtils.doPost2("http://192.168.0.100:38080/FangJiaDataService/saveData", jsonStr)
|
||||
if (json.status == 0) {
|
||||
return "³É¹¦"
|
||||
}else if(json.status == 1){
|
||||
return json.msg
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+629
-10
@@ -4,14 +4,25 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.FiledStrand
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 写字楼字段清洗入库(自动入库)
|
||||
@@ -19,6 +30,28 @@ import java.util.concurrent.Executors
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
//"build_date":"build_year", "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address",
|
||||
// "live_date":"live_date", "loop_line_info":"loop_line"
|
||||
def fieldsMap = ["estate_charge":"property_fee","block":"block","estate_company":"property_company","developer":"developer",
|
||||
"virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area"
|
||||
,"finish_building_date":"finish_building_date","property_life":"property_life","standard_area":"standard_area","total_floor":"total_floor",
|
||||
"standard_height":"floor_height", "real_use_percent":"room_rate", "loop_line_info":"ring_location","car_site_count":"parking",
|
||||
"parking_fee":"car_fee","elevators_count":"elevator_num","bay_area":"open_area"]//key:seaweed表字段 value:es表字段
|
||||
|
||||
//"安居客":"安居客app",
|
||||
def sourceMap = ["房天下":"房天下","楼盘网":"楼盘网","办办网":"办办网","点点租":"点点租"]
|
||||
|
||||
def fieldMap = ["编号":"_id","城市":"city","区域":"region","名称":"name","别名":"alias","租金":"rent_price","板块":"block","物业公司":"estate_company","物业费":"estate_charge",
|
||||
"地址":"address","地址别名":"addr_alias","经度":"lng","纬度":"lat","类型":"estate_type2","建筑面积":"build_area","等级":"bed_kind",
|
||||
"开发商":"developer","占地面积":"all_area","标准层面积":"standard_area","产权年限":"property_life","总层数":"total_floor","是否涉外":"is_foreign",
|
||||
"容积率":"cubage_percent","绿化率":"virescence_percent","货梯数":"goods_elevator_count","竣工时间":"finish_building_date","客梯数":"person_elevators_count",
|
||||
"标准层高":"standard_height","净高":"clear_height","大堂层高":"lobby_height","得房率":"real_use_percent","是否可分割":"divisible","环线位置":"loop_line_info",
|
||||
"车位描述":"car_site","车位数量":"car_site_count","地上车位":"land_car_site","地下停车位":"underground_car_site","车位租金":"parking_fee","电梯数量":"elevators_count",
|
||||
"是否注册":"is_register","开间面积":"bay_area","楼层信息":"floor_info"]
|
||||
|
||||
static DBCollection BLOCKPOLYLINECOLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border2", "mongo.db.fangjia.com")
|
||||
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static main(args) {
|
||||
@@ -27,12 +60,182 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
for(def c : citys){
|
||||
def offices = findAllOffice(c)
|
||||
strandField(offices)
|
||||
// def str = "1200平方/层,1067㎡,1650-1900平米"
|
||||
|
||||
|
||||
// def citys = CommonDatautils.findAllCity().reverse()
|
||||
// for(def c : citys){
|
||||
// def offices = findAllOffice(c)
|
||||
// strandField(offices)
|
||||
// }
|
||||
|
||||
// addBlock()
|
||||
|
||||
// addFieldValue()
|
||||
|
||||
// updateErrorField()
|
||||
export()
|
||||
test()
|
||||
// addOffice()
|
||||
}
|
||||
|
||||
def export(){
|
||||
def offices = CommonDatautils.findAllFiledDistrict("office", "上海")
|
||||
def fields = fieldMap.keySet().asList()
|
||||
def rets = []
|
||||
rets << fields
|
||||
for(def i=0;i<offices.size();i++){
|
||||
def data = offices[i]
|
||||
if(data.region == "浦东"){
|
||||
def row = []
|
||||
fieldMap.each{k,v->
|
||||
def val = data.get(v)
|
||||
if(k == "编号"){
|
||||
row << val.toString()
|
||||
}else if(v=="lat" || v=="lng"){
|
||||
row << val.toString().toDouble()/10000000000.0
|
||||
}else if(v == "divisible"){
|
||||
|
||||
if(val==1){
|
||||
row << "是"
|
||||
}else if(val==0){
|
||||
row << "否"
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
}else if(v.contains("alias")){
|
||||
if(val){
|
||||
row << val.join(" ")
|
||||
}else{
|
||||
row << ''
|
||||
}
|
||||
}else{
|
||||
row << val
|
||||
}
|
||||
}
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\写字楼\\中远.xlsx",rets, "数据")
|
||||
}
|
||||
|
||||
def test(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\写字楼\\写字楼价格租金.xlsx",0 , 1)
|
||||
def map = [:]
|
||||
for(def d : datas){
|
||||
map.put(d['A']+"#"+d['B']+'#'+d['C'], d)
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\中远.xlsx",{d->
|
||||
def row = map.get(d['B']+"#"+d['C']+'#'+d['D'])
|
||||
d['F'] = row['F']
|
||||
})
|
||||
}
|
||||
|
||||
def updateErrorField(){
|
||||
// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
|
||||
def offices = CommonDatautils.findAllDistricts("上海","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"])
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
|
||||
for(def i=0;i<offices.size();i++){
|
||||
final def o = offices[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
updateDistField(o)
|
||||
println totalCount.get() +" "+o.city
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def updateDistField(o){
|
||||
def addr_alias = o.addr_alias
|
||||
def udata = new BasicDBObject()
|
||||
if(o.property_life){
|
||||
def newVal = FiledStrand.processPropertyLife(o.property_life)
|
||||
udata.put("property_life", newVal)
|
||||
}
|
||||
|
||||
if(o.total_floor){
|
||||
def newVal = FiledStrand.processFloor(o.total_floor)
|
||||
udata.put("total_floor", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.finish_building_date){
|
||||
def newVal = FiledStrand.processFinishBuildingDate(o.finish_building_date)
|
||||
udata.put("finish_building_date", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.real_use_percent){
|
||||
def newVal = FiledStrand.processRealUsePercent(o.real_use_percent)
|
||||
udata.put("real_use_percent", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.underground_car_site){
|
||||
def newVal = FiledStrand.processCarSite(o.underground_car_site)
|
||||
udata.put("underground_car_site", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.land_car_site){
|
||||
def newVal = FiledStrand.processCarSite(o.land_car_site)
|
||||
udata.put("land_car_site", newVal.toString())
|
||||
}
|
||||
|
||||
if(o.parking_fee){
|
||||
def newVal = FiledStrand.processCarSiteRent(o.parking_fee)
|
||||
udata.put("parking_fee", newVal)
|
||||
}
|
||||
|
||||
if(addr_alias){
|
||||
def addrs = []
|
||||
for(def arr : addr_alias){
|
||||
def addr = FiledStrand.processStandardAddress(arr, o.city)
|
||||
if(addr){
|
||||
addrs << addr
|
||||
}
|
||||
}
|
||||
if(addrs!=addr_alias){
|
||||
if(addrs){
|
||||
udata.put("addr_alias", addrs)
|
||||
}else{
|
||||
udata.put("addr_alias", null)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// if(o.estate_charge && !o.estate_charge.isNumber()){
|
||||
//// def val = o.estate_charge.toString().toDouble()
|
||||
//// if(val>40 || val < 0.01){
|
||||
//// println val
|
||||
// udata.put("estate_charge", null)
|
||||
//// }
|
||||
// }
|
||||
|
||||
if(udata){
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',o._id),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def addFieldValue(){
|
||||
def citys = ["上海"]
|
||||
def analysRets = []
|
||||
for(def c : citys){
|
||||
def totalCount = CommonDatautils.findAllDistrictCount("office", c)
|
||||
if(totalCount){
|
||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -65,15 +268,431 @@ class OfficeFieldCleanBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def findAllOffice(city){
|
||||
def sql = "select * from office where city='${city}'"
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery(sql, { d->
|
||||
rets << d
|
||||
def findEsDatas(c, esWebSource){
|
||||
def list = []
|
||||
ElasticUtils.executeSqlQuery("select * from office where city='${c}' and source='${esWebSource}'", {r->
|
||||
list << r
|
||||
})
|
||||
return list
|
||||
}
|
||||
|
||||
|
||||
def matchDist(city, region, name){
|
||||
def dist = CommonDatautils.findDistrict(city, region, name, "office")
|
||||
return dist;
|
||||
}
|
||||
|
||||
def matchApi(esRets){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(esRets.size())
|
||||
for(def i=0;i<esRets.size();i++){
|
||||
final def data = esRets[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = matching(data)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def matching(data){
|
||||
def name = data.name
|
||||
if(data.name && data.name.contains("·")){
|
||||
name = data.name.replace("·", "")
|
||||
}
|
||||
|
||||
def flag = "名称匹配"
|
||||
def dist = matchDist(data.city, data.region, name)
|
||||
|
||||
if(!dist){
|
||||
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"office")
|
||||
flag = "映射别名匹配"
|
||||
}
|
||||
if(!dist) {
|
||||
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "office")
|
||||
if(rets && rets.size()==1){
|
||||
flag = "别名匹配"
|
||||
dist = rets[0]
|
||||
}
|
||||
}
|
||||
|
||||
if(dist){
|
||||
data.flag = flag
|
||||
data.match_id = dist._id.toString()
|
||||
data.match_region = dist.region
|
||||
data.match_name = dist.name
|
||||
data.credit = 0.95
|
||||
}else{
|
||||
def jsonObject = OpenApiClient.addressMatch(data.city, name, "office", false,data.region, null)
|
||||
if(jsonObject!=null && jsonObject.precise==-1){
|
||||
if(data.address && isStandardRoad(data.address)){
|
||||
jsonObject = OpenApiClient.addressMatch(data.city, data.address, "office", false,data.region, name)
|
||||
}else{
|
||||
data.flag = "匹配接口-地址"
|
||||
}
|
||||
}else{
|
||||
data.flag = "匹配接口-名称"
|
||||
}
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
data.match_id = jsonObject.get("id")
|
||||
data.match_region = jsonObject.get("district")
|
||||
data.match_name = jsonObject.get("name")
|
||||
data.credit = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
if(data.match_id){
|
||||
return data
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
def selectRestByCredit(matchRets){
|
||||
def matchRets1 = []
|
||||
for(def r : matchRets){
|
||||
if(r.credit && r.credit>=0.9){
|
||||
matchRets1 << r
|
||||
}
|
||||
}
|
||||
return matchRets1
|
||||
}
|
||||
|
||||
|
||||
def addFieldFromMatch(c, totalCount, analysRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
|
||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||
def esRets = findEsDatas(c, esWebSource)
|
||||
// esRets = esRets[0..100]
|
||||
def matchRets1 = matchApi(esRets)
|
||||
def matchRets2 = selectRestByCredit(matchRets1)
|
||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||
|
||||
def updateRets = processFieldsFromMatch(c , matchRets2)
|
||||
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+ DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
|
||||
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||
println c +" "+precentMap.toString()
|
||||
updateDB(c, updateRets)
|
||||
println "------------------------------------------------------------------------------"
|
||||
def pre1 = 0
|
||||
def pre2 = 0
|
||||
if(esRets){
|
||||
pre1 = DataHandleUtils.getDoubleRound(2, matchRets1.size()*100/esRets.size())
|
||||
pre2 = DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/esRets.size())
|
||||
}
|
||||
def analyRet = []
|
||||
analyRet << seaweedmapSource
|
||||
analyRet << c
|
||||
fields.each{key->
|
||||
analyRet << precentMap.get(key)
|
||||
}
|
||||
analyRet << esRets.size()
|
||||
analyRet << matchRets1.size()
|
||||
analyRet << matchRets2.size()
|
||||
analyRet << pre1
|
||||
analyRet << pre2
|
||||
analysRets << analyRet
|
||||
}
|
||||
return analysRets;
|
||||
}
|
||||
|
||||
def fieldUpdate(total,updateRets){
|
||||
|
||||
def countMap = [:]
|
||||
|
||||
for(def rm : updateRets){
|
||||
def valMap = rm.value
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
if(valMap.containsKey(seaweedfield)) {
|
||||
if(countMap.containsKey(seaweedfield)) {
|
||||
countMap.put(seaweedfield, countMap.get(seaweedfield) + 1)
|
||||
}else{
|
||||
countMap.put(seaweedfield, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println countMap.toString()
|
||||
def precentMap = [:]
|
||||
fieldsMap.each() {key,skey->
|
||||
if(countMap.containsKey(key)) {
|
||||
precentMap.put(key, DataHandleUtils.getDoubleRound(2, countMap.get(key)*100/total))
|
||||
}else {
|
||||
precentMap.put(key, 0)
|
||||
}
|
||||
}
|
||||
return precentMap;
|
||||
}
|
||||
|
||||
|
||||
def processFieldsFromMatch(c,matchRets){
|
||||
def fields = fieldsMap.keySet() as List
|
||||
fields << "city"
|
||||
fields << "estate_type2"
|
||||
def fieldValueMap = [:]
|
||||
for(def matchDist : matchRets){
|
||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||
if(!dist){
|
||||
println c+" "+matchDist.match_id+" 不存在"
|
||||
continue
|
||||
}
|
||||
def newValues = strandField(c, dist, matchDist)
|
||||
if(newValues){
|
||||
println dist._id.toString()+" "+newValues.toString()
|
||||
fieldValueMap.put(dist._id.toString(), newValues)
|
||||
}
|
||||
}
|
||||
|
||||
return fieldValueMap;
|
||||
}
|
||||
|
||||
def strandField(c,dist, matchDist){
|
||||
def newValues = [:]
|
||||
fieldsMap.each{seaweedfield,esField->
|
||||
def val = dist.get(seaweedfield)
|
||||
if(!val){
|
||||
def esVal = matchDist.get(esField)
|
||||
if(esVal){
|
||||
def newVal = null
|
||||
if(seaweedfield == "estate_charge"){
|
||||
newVal = FiledStrand.processEstateCharge(c, FiledStrand.getNumber(esVal), dist.estate_type2)
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
if(seaweedfield == "build_date" || seaweedfield == "live_date"){
|
||||
newVal = FiledStrand.processFinishBuildingDate(esVal)
|
||||
}
|
||||
if(seaweedfield == "estate_company" || seaweedfield == "developer"){
|
||||
newVal = FiledStrand.processCompany(esVal)
|
||||
}
|
||||
if(seaweedfield =="virescence_percent"){
|
||||
newVal = FiledStrand.processVirescencePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="cubage_percent"){
|
||||
newVal = FiledStrand.processCubagePercent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="all_area" || seaweedfield =="build_area"){
|
||||
newVal = FiledStrand.processArea(esVal.toString())
|
||||
if(newVal){
|
||||
newVal = newVal.toString()
|
||||
}
|
||||
}
|
||||
if(seaweedfield =="household_count"){
|
||||
newVal = FiledStrand.processHouseholdCount(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
if(seaweedfield =="loudong_count"){
|
||||
newVal = FiledStrand.processLoudongCount(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="car_site_count"){
|
||||
newVal = FiledStrand.processCarSite(esVal)
|
||||
}
|
||||
if(seaweedfield =="car_site"){
|
||||
newVal = FiledStrand.processCarSiteDesc(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="property_life"){
|
||||
newVal = FiledStrand.processPropertyLife(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield =="alias"){
|
||||
newVal = FiledStrand.processAlias(esVal, dist.name, dist.alias)
|
||||
}
|
||||
|
||||
if(seaweedfield =="address"){
|
||||
newVal = FiledStrand.processAddress(esVal, dist.city)
|
||||
}
|
||||
if(seaweedfield =="block"){
|
||||
newVal = FiledStrand.processBlock(dist.city, dist.region, esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_type"){
|
||||
newVal = FiledStrand.processBuildType(esVal, dist.estate_type2)
|
||||
}
|
||||
|
||||
if(seaweedfield == "land_property"){
|
||||
newVal = FiledStrand.processlandProperty(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "build_structure"){
|
||||
newVal = FiledStrand.processBuildStructure(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "total_floor"){
|
||||
newVal = FiledStrand.processFloor(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "real_use_percent"){
|
||||
newVal = FiledStrand.processRealUsePercent(esVal)
|
||||
}
|
||||
if(seaweedfield == "parking_fee"){
|
||||
newVal = FiledStrand.processCarSiteRent(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "elevators_count"){
|
||||
newVal = FiledStrand.processElevators(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "standard_area"){
|
||||
newVal = FiledStrand.processStandardArea(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "standard_height"){
|
||||
newVal = FiledStrand.processStandardHeight(esVal)
|
||||
}
|
||||
|
||||
if(seaweedfield == "bay_area"){
|
||||
newVal = esVal
|
||||
}
|
||||
|
||||
if(newVal){
|
||||
newValues.put(seaweedfield, newVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return newValues;
|
||||
}
|
||||
|
||||
def updateDB(c, updateRets){
|
||||
|
||||
for(def rm : updateRets) {
|
||||
def id = rm.key
|
||||
def valMap = rm.value
|
||||
|
||||
BasicDBObject udata = new BasicDBObject()
|
||||
valMap.each {k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
def addBlock(){
|
||||
def rets = findAllBlockRange()
|
||||
|
||||
def offices = CommonDatautils.findAllDistricts("上海", "office", ["city","region","name","block","lat2", "lng2"])
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(offices.size())
|
||||
|
||||
|
||||
for(def i=0;i<=offices.size();i++){
|
||||
final def office = offices[i]
|
||||
if(office && !office.block){
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def lastObj = null
|
||||
for (def r : rets) {
|
||||
def flag = InsidePolygon.isInArea(office.lng2 / 10000000000.0, office.lat2 / 10000000000.0, r.points)
|
||||
if (flag && r.name) {
|
||||
lastObj = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (lastObj) {
|
||||
def udata = [:]
|
||||
udata.put("block", lastObj.name)
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "office")
|
||||
udata.put("username", "高娟")
|
||||
udata.put("id", office._id.toString())
|
||||
udata.put("loopLineInfo", lastObj.loop)
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println s
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
latch.await()
|
||||
}
|
||||
|
||||
def findAllBlockRange(){
|
||||
def rets = []
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = BLOCKPOLYLINECOLL.find(q)
|
||||
for(def d : dbcursor) {
|
||||
def pointList = InsidePolygon.createPoint3(d.border);
|
||||
d.points = pointList
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name, datas){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category","office");
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", "update")
|
||||
obj.put("data", datas)
|
||||
obj.put("user_name", "高娟")
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
def addOffice(){
|
||||
def city = "上海"
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\写字楼\\bd_file_insert(待入库).xlsx",{d->
|
||||
def udata = [:]
|
||||
def formatAddress = AddressUtil.formatAddress(city, d['B'])
|
||||
udata.put("city", city)
|
||||
udata.put("region", d['A'])
|
||||
udata.put("name", d['D'])
|
||||
udata.put("address", formatAddress)
|
||||
long lng = (d['L'].toString().toDouble()*10000000000).longValue()
|
||||
long lat = (d['K'].toString().toDouble()*10000000000).longValue()
|
||||
|
||||
udata.put("lat", lat)
|
||||
udata.put("lng", lng)
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "office")
|
||||
udata.put("username", "高娟")
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
println d['D'] + s
|
||||
})
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+5
-2
@@ -10,6 +10,9 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
*处理地址在地址别名中有重复
|
||||
*/
|
||||
class SameDistName {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
@@ -18,13 +21,13 @@ class SameDistName {
|
||||
def task = new SameDistName()
|
||||
// task.run()
|
||||
//地址和地址别名完全一致
|
||||
// task.addressPocess()
|
||||
task.addressPocess()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addressPocess(){
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
def citys = ["福州"]//CommonDatautils.findAllCity();
|
||||
|
||||
for(def c : citys){
|
||||
def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "name", "address","addr_alias"])
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.fangjia.script.gj.house
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class RemoveHouse extends BaseBatch{
|
||||
|
||||
DBCollection SEABIRD_COLL = MongoUtils.getCollectionByName("fangjia", "seabird", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new RemoveHouse(), args)
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def start = new Date()
|
||||
def endDate = DateUtils.parseFromTightYYYYMMDD("2023-02-01")
|
||||
def q = new BasicDBObject()
|
||||
q.put("c_date", new BasicDBObject('$lte', endDate))
|
||||
def dbcursor = SEABIRD_COLL.find(q).batchSize(2000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d._id
|
||||
if(rets.size()==200){
|
||||
def delquery = new BasicDBObject()
|
||||
delquery.put("_id", new BasicDBObject('$in', rets))
|
||||
SEABIRD_COLL.remove(delquery)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
}
|
||||
def end = new Date()
|
||||
|
||||
println "¿ªÊ¼Ê±¼ä£º"+start
|
||||
println "½áÊøÊ±¼ä£º"+end
|
||||
}
|
||||
}
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 判断小区是否在城市范围内
|
||||
*/
|
||||
class DistrictLocationBatch {
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
static DBCollection RANGECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictLocationBatch()
|
||||
// task.run()
|
||||
task.deleteDist()
|
||||
}
|
||||
|
||||
def deleteDist(){
|
||||
ExcelUtils.multiThreadingModify("d:\\数据\\小区\\非本城市小区.xlsx",{d->
|
||||
DBObject q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("region", d['B'])
|
||||
q.put("name", d['C'])
|
||||
q.put("cat", 'district')
|
||||
q.put("status", 0)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
})
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def cityMap = findCityRange()
|
||||
def allrets = []
|
||||
for (def city : citys) {
|
||||
def points = cityMap.get(city)
|
||||
def pointList = InsidePolygon.createPoint3(points);
|
||||
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "lat", "lng"])
|
||||
def rets = findNotInCityDistricts(city, disticts, pointList)
|
||||
allrets.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\小区\\非本城市小区.xlsx", allrets, "经纬度不在此城市中")
|
||||
}
|
||||
|
||||
def findNotInCityDistricts(city, districts, pointList){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(dist.lng){
|
||||
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
|
||||
if(!flag) {
|
||||
def ret = []
|
||||
ret << dist.city
|
||||
ret << dist.region
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
println ret
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
latch.await()
|
||||
println city+" "+ rets.size() +" "+districts.size()
|
||||
return rets
|
||||
}
|
||||
|
||||
def findCityRange(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = RANGECOLL.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
cityMap.put(d.fjw_city, d.polyline)
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
def isInArea(lng, lat, pointList){
|
||||
def flag = false;
|
||||
for(def pointArr : pointList){
|
||||
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
|
||||
if(flag){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
|
||||
}
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
class GaodeCityRange {
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
|
||||
static main(args) {
|
||||
def task = new GaodeCityRange()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
|
||||
def citys1= ["上海","北京","广州","深圳","成都","天津","西安","杭州","沈阳","重庆","武汉","南京","石家庄","福州","宁波","贵阳","苏州","长沙","大连","昆明","厦门","太原","无锡","郑州","呼和浩特","济南","哈尔滨","兰州","青岛","佛山","长春","东莞","合肥","烟台","乌鲁木齐","南昌","泉州","齐齐哈尔","包头","常德","大同","温州","银川","西宁","扬州","绵阳","威海","吉林","中山","邯郸","湛江","潍坊","保定","常州","秦皇岛","嘉兴","鞍山","马鞍山","宜昌","徐州","株洲","大庆","镇江","芜湖","赣州","南通","九江","洛阳","泰州","惠州","台州","湖州","绍兴","江门","海口","珠海","金华","桂林","连云港","马鞍山","淄博","柳州","盐城","汕头","衡阳","揭阳","茂名","牡丹江","平顶山","本溪","安阳","舟山","岳阳","许昌","四平","淮南","淮安","赤峰","佳木斯","南阳","临沂","临汾","榆林","三亚","唐山","咸阳","宜宾","自贡","廊坊","南宁","攀枝花","襄阳","延安","丽江","大理","湘潭","六盘水","清远","衢州","宿迁","北海","莆田","玉林","张家界","肇庆","曲靖","新乡","营口","潮州","河源","黄石","焦作","盘锦","韶关","阳江","梅州","云浮","玉溪","遵义","中卫","武威","铁岭","天水","泰安","黄冈","广元","阜阳","定西","辽源","丽水","乐山","菏泽","锦州","汉中","黄山","白山","安顺","安康","绥化","钦州","鄂尔多斯","梧州","百色","鸡西","七台河","白城","朝阳","沧州","东营","德州","亳州","安庆","六安","宁德","三沙","临沧","拉萨","南充","泸州","资阳","乌海","邵阳","克拉玛依","巴彦淖尔","荆门","来宾","开封","信阳","固原","驻马店","防城港","朔州","铜川","呼伦贝尔","双鸭山","平凉","崇左","白银","河池","吉安","通化","景德镇","鹤岗","海东","黑河","阜新","抚顺","葫芦岛","张家口","衡水","邢台","石嘴山","济宁","日照","聊城","滨州","淮北","宣城","三明","宿州","龙岩","保山","池州","铜仁","乌兰察布","南平","内江","达州","贺州","巴中","贵港","昭通","怀化","鄂州","伊春","毕节","十堰","遂宁","漯河","松原","眉山","鹤壁","忻州","郴州","晋中","益阳","辽阳","宝鸡","酒泉","张掖","随州","承德","鹰潭","新余","孝感","枣庄","周口","濮阳","阳泉","运城","蚌埠","渭南","滁州","铜陵","嘉峪关","漳州","庆阳","汕尾","上饶","普洱","宜春","德阳","广安","雅安","永州","娄底","荆州","咸宁","三门峡","商丘","长治","吕梁","商洛","金昌","陇南","萍乡","抚州","吴忠","通辽","晋城","阿克苏","喀什","昌吉","楚雄","大兴安岭","吐鲁番","哈密","和田","塔城","阿勒泰","博尔塔拉","巴音郭楞","克孜勒苏","伊犁","红河","昌都","日喀则","山南","那曲","阿里","林芝","兴安盟","锡林郭勒盟","阿拉善盟","临夏","甘南","黔西南","黔东南","黔南","恩施","湘西","延边","昆玉","丹东","石河子","阿拉尔","图木舒克","五家渠","济源","仙桃","潜江","天门","神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","北屯","铁门关","双河","可克达拉"]
|
||||
HttpRequest request = new HttpRequest()
|
||||
def rows = []
|
||||
|
||||
citys.each{c->
|
||||
if(!citys1.contains(c)){
|
||||
c = c.replace("州","")
|
||||
JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
|
||||
if(resp.containsKey("districts")){
|
||||
def datas = resp.get("districts")
|
||||
for(def d : datas){
|
||||
println c +" "+ d.name+" "+d.level
|
||||
if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){
|
||||
rows << d
|
||||
}else{
|
||||
if(d.level=="city"){
|
||||
rows << d
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
// citys.each{c->
|
||||
// println c
|
||||
// JSONObject resp = request.getJsonContent("http://restapi.amap.com/v3/config/district?key=4ccfe3c898a9489884bd2bcb1ef6dcde&keywords=${c}&subdistrict=0&extensions=all", null, "UTF-8")
|
||||
// if(resp.containsKey("districts")){
|
||||
// def datas = resp.get("districts")
|
||||
// for(def d : datas){
|
||||
// println c +" "+ d.name+" "+d.level
|
||||
// if(["上海市","北京市","天津市","重庆市"].contains(d.name.toString())){
|
||||
// rows << d
|
||||
// }else{
|
||||
// if(d.level=="city"){
|
||||
// rows << d
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
if(rows){
|
||||
rows.each {r->
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("level", r.level)
|
||||
obj.put("center", r.center)
|
||||
obj.put("center", r.center)
|
||||
obj.put("polyline", r.polyline)
|
||||
obj.put("name", r.name)
|
||||
obj.put("adcode", r.adcode)
|
||||
obj.put("citycode", r.citycode)
|
||||
|
||||
def city = RegionData.formatCity(r.name)
|
||||
obj.put("fjw_city", city)
|
||||
POLYLINECOLL.save(obj)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+127
-13
@@ -3,6 +3,9 @@ package com.fangjia.script.gj.location
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
@@ -25,28 +28,122 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
|
||||
class LocationTest {
|
||||
/**
|
||||
* 根据区域围栏修正区域
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class RegionLocationUpdateBatch extends BaseBatch{
|
||||
|
||||
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
DBCollection OPERATION_LOG_COLL = MongoUtils.getCollectionByName("fangjia", "operation_log", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
static main(args) {
|
||||
// Date date=new Date(1622802609);
|
||||
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||
// String str = format.format(date);
|
||||
// System.out.println(str);
|
||||
runBatch(new RegionLocationUpdateBatch(), args);
|
||||
}
|
||||
|
||||
def task = new LocationTest()
|
||||
task.run()
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
regionUpdate()
|
||||
// task.run()
|
||||
// task.run2()
|
||||
// task.run3()
|
||||
// task.run4()
|
||||
// task.run5()
|
||||
}
|
||||
|
||||
def regionUpdate(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
|
||||
def datas = findInRegionDistricts(city, districts, regionPointsMap)
|
||||
println "【${index}】"+city +" 经纬度处理完成"
|
||||
rets.addAll(datas)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\小区\\区域纠正.xlsx", rets, "数据")
|
||||
}
|
||||
|
||||
|
||||
def findInRegionDistricts(city, districts, regionPointsMap){
|
||||
def index = 0
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def regionMap = [:]
|
||||
regionPointsMap.each{k,v->
|
||||
def pointList = InsidePolygon.createPoint3(v);
|
||||
regionMap.put(k, pointList)
|
||||
}
|
||||
def keys = regionPointsMap.keySet()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
if(dist.lng){
|
||||
totalCount.addAndGet(1)
|
||||
def pointList = regionMap.get(dist.region)
|
||||
def flag = InsidePolygon.isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
|
||||
def newRegion = null
|
||||
if(!flag){
|
||||
for(def m : regionPointsMap){
|
||||
if(m.key != dist.region){
|
||||
pointList = regionMap.get(m.key)
|
||||
flag = InsidePolygon.isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, pointList)
|
||||
if(flag){
|
||||
newRegion = m.key
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(flag){
|
||||
if(newRegion){
|
||||
println dist.city +" "+dist.region+" "+dist.name+" 【${newRegion}】"
|
||||
def udata = [:]
|
||||
udata.put("region", newRegion)
|
||||
udata.put("status", "0")
|
||||
udata.put("cat", "district")
|
||||
udata.put("username", "高娟")
|
||||
udata.put("id", dist._id.toString())
|
||||
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
|
||||
def opt = "update"
|
||||
if(s && s!='成功'){
|
||||
if(s && s.contains("该区域小区已存在")){
|
||||
def delUdata = new BasicDBObject()
|
||||
delUdata.put("status", 1)
|
||||
delUdata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dist._id), new BasicDBObject('$set', delUdata))
|
||||
saveOptLog(dist.city, dist.region, dist.name)
|
||||
opt = "delete"
|
||||
}
|
||||
}
|
||||
|
||||
rets << [dist.city, dist.region, dist.name, newRegion, opt]
|
||||
}
|
||||
}else{
|
||||
println dist.city +" "+dist.region+" "+dist.name+" 没有找到对应的区域"
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "º¼ÖÝ"
|
||||
def regionPointsMap = findRegionRange(city)
|
||||
@@ -446,12 +543,11 @@ class LocationTest {
|
||||
|
||||
def findRegionRange(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",city+"ÊÐ")
|
||||
q.put("city",city)
|
||||
def dbcursor = POLYLINECOLL.find(q)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def region = RegionData.formatRegion(city,d.region)
|
||||
regionMap.put(region, d.polyline)
|
||||
regionMap.put(d.fjw_region, d.polyline)
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
@@ -525,4 +621,22 @@ class LocationTest {
|
||||
return rets
|
||||
}
|
||||
|
||||
def saveOptLog(city, region, name){
|
||||
def obj = new BasicDBObject();
|
||||
obj.put("city",city);
|
||||
obj.put("region",region);
|
||||
obj.put("district_name",name);
|
||||
obj.put("category","district");
|
||||
obj.put("c_date", new Date())
|
||||
obj.put("m_date", new Date())
|
||||
obj.put("type", "remove")
|
||||
obj.put("user_name", "高娟")
|
||||
obj.put("origin", "script")
|
||||
OPERATION_LOG_COLL.save(obj)
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+23
-18
@@ -1,25 +1,19 @@
|
||||
package com.fangjia.script.gj
|
||||
package com.fangjia.script.gj.location
|
||||
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.collections.CollectionUtils
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.bson.Document
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 抓取区域边框
|
||||
* 高德抓取区域边框
|
||||
*/
|
||||
class RegionPointCrawl {
|
||||
|
||||
@@ -28,13 +22,19 @@ class RegionPointCrawl {
|
||||
|
||||
static main(args) {
|
||||
def task = new RegionPointCrawl();
|
||||
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
|
||||
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
|
||||
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
|
||||
def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"
|
||||
//23个省 河北省、山西省、吉林省、辽宁省、黑龙江省、陕西省、甘肃省、青海省、山东省、福建省、浙江省、台湾省、河南省、湖北省、湖南省、江西省、江苏省、安徽省、广东省、海南省、四川省、贵州省、云南省
|
||||
//4个直辖市 北京市、上海市、天津市、重庆市
|
||||
//5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"
|
||||
// def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"]
|
||||
def provinces = ["甘肃省"]
|
||||
def index = 0
|
||||
for(def p : provinces){
|
||||
index ++
|
||||
println index +"/"+provinces.size() +" "+p
|
||||
task.crawlCityAndDistrictCode(p);
|
||||
println "城市 区域 code 完成"
|
||||
task.run(p);
|
||||
println "区域围栏 完成"
|
||||
}
|
||||
|
||||
}
|
||||
@@ -43,8 +43,8 @@ class RegionPointCrawl {
|
||||
crawlCity(province)
|
||||
}
|
||||
|
||||
def run(city){
|
||||
def cityCodes = findCityCode(city);
|
||||
def run(pro){
|
||||
def cityCodes = findCityCode(pro);
|
||||
if(cityCodes){
|
||||
for(def cityMap : cityCodes){
|
||||
def codeMap = findAllDistrictCode(cityMap.city_code);
|
||||
@@ -59,7 +59,8 @@ class RegionPointCrawl {
|
||||
if(ret.size()>0){
|
||||
def standardCity = RegionData.formatCity(cityMap.name)
|
||||
if(standardCity){
|
||||
saveData(standardCity, m.name, m.ad_code , ret);
|
||||
def newRegion = RegionData.formatRegion(standardCity, m.name)
|
||||
saveData(pro, standardCity, m.name, m.ad_code , ret, newRegion);
|
||||
}else{
|
||||
println cityMap.name +" ûÓбê×¼»¯"
|
||||
}
|
||||
@@ -102,7 +103,7 @@ class RegionPointCrawl {
|
||||
def findCityCode(city){
|
||||
Pattern pattern = Pattern.compile(city, Pattern.CASE_INSENSITIVE);
|
||||
def q = null;
|
||||
if(city.contains("省") || city.endsWith("自治区")){
|
||||
if(city.contains("省") || city.endsWith("自治区") || ["上海市","天津市","北京市","重庆市"].contains(city)){
|
||||
q = new BasicDBObject("province", pattern);
|
||||
}else{
|
||||
q = new BasicDBObject("name", pattern);
|
||||
@@ -156,18 +157,22 @@ class RegionPointCrawl {
|
||||
}
|
||||
}
|
||||
|
||||
def saveData(city, region, code, ret){
|
||||
def saveData(pro, city, region, code, ret, formatRegion){
|
||||
def q = new BasicDBObject()
|
||||
q.put("code", code)
|
||||
def dbo = POLYLINECOLL.findOne(q)
|
||||
if(!dbo){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", pro)
|
||||
o.put("city", city)
|
||||
o.put("region", region)
|
||||
o.put("code", code)
|
||||
o.put("center", ret.center.get(0))
|
||||
o.put("level", ret.level.get(0))
|
||||
o.put("polyline", ret.polyline.get(0));
|
||||
if(formatRegion){
|
||||
o.put("fjw_region", formatRegion)
|
||||
}
|
||||
POLYLINECOLL.save(o);
|
||||
}
|
||||
}
|
||||
+30
-18
@@ -33,7 +33,7 @@ public class BeiKeCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
|
||||
|
||||
//广州,佛山 重庆
|
||||
@@ -204,8 +204,11 @@ public class BeiKeCrawlByChrome {
|
||||
if(dealDate && dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "贝壳找房")
|
||||
|
||||
if (!dbo) {
|
||||
parse(city, e, region, driver)
|
||||
parse(city, e, region, driver, null)
|
||||
}else if(dbo && !dbo.lift){
|
||||
parse(city, e, region, driver, dbo._id)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
@@ -224,7 +227,7 @@ public class BeiKeCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e, region, driver){
|
||||
def parse(city,e, region, driver, id){
|
||||
try {
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
|
||||
@@ -289,20 +292,26 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
detail.source = "贝壳找房"
|
||||
|
||||
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
|
||||
def detailDiv1 = JQueryUtils.select(".introContent .base", detailDoc)
|
||||
def detailDiv2 = JQueryUtils.select(".introContent .transaction", detailDoc)
|
||||
|
||||
def liftStr = detailDiv1.select("li:contains(梯户比例)").text().replace("梯户比例","")
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
}
|
||||
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("㎡",""))
|
||||
detail.room_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(户型结构)").text().replace("户型结构",""))
|
||||
detail.elevator = DataFormatUtils.getValue(detailDiv1.select("li:contains(配备电梯)").text().replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑结构)").text().replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(detailDiv1.select("li:contains(建成年代)").text().replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(detailDiv1.select("li:contains(建筑类型)").text().replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(detailDiv1.select("li:contains(所在楼层)").text().replace("㎡",""))
|
||||
|
||||
detail.trade_property = detailDiv2.select("li:contains(交易权属)").text().replace("交易权属","")
|
||||
detail.property_ownership = DataFormatUtils.getValue(detailDiv2.select("li:contains(房权所属)").text().replace("房权所属",""))
|
||||
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
|
||||
detail.house_use = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋用途)").text().replace("房屋用途",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(detailDiv2.select("li:contains(房屋年限)").text().replace("房屋年限",""))
|
||||
if(detail.total_price && detail.avg_price){
|
||||
|
||||
detail.url = detailUrl
|
||||
@@ -323,8 +332,11 @@ public class BeiKeCrawlByChrome {
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
|
||||
if(id){
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
|
||||
}else{
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
}
|
||||
}
|
||||
}catch(Exception ex){
|
||||
@@ -453,9 +465,9 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = "https:"+e.attr("href")
|
||||
// if (citys.contains(e.text())){
|
||||
if (citys.contains(e.text())){
|
||||
list << [e.text(), href]
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return list
|
||||
@@ -483,7 +495,7 @@ public class BeiKeCrawlByChrome {
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(20000)
|
||||
Thread.sleep(10000)
|
||||
}catch(Exception e){
|
||||
Thread.sleep(15000)
|
||||
}
|
||||
|
||||
+50
-25
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.gj.paimai
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
@@ -32,7 +33,7 @@ public class LianjiaCrawlByChrome {
|
||||
def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
|
||||
def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -2, "yyyy-MM")
|
||||
|
||||
static main(args) {
|
||||
def task = new LianjiaCrawlByChrome();
|
||||
@@ -297,16 +298,23 @@ public class LianjiaCrawlByChrome {
|
||||
def dealDate = e.select(".dealDate").text()
|
||||
dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern(startMonth + "-01", "yyyy-MM-dd")
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver)
|
||||
if(dealDate){
|
||||
if(dealDate.getTime()>=endDate.getTime()){
|
||||
detailUrls << durl
|
||||
def dbo = findTrade(durl , "链家在线")
|
||||
if (!dbo) {
|
||||
parse(city, e, regionText, driver, null)
|
||||
}else if(dbo && !dbo.lift){
|
||||
parse(city, e, regionText, driver, dbo._id)
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
}
|
||||
}else{
|
||||
stop = true;
|
||||
break;
|
||||
println url +" 成交时间有误!"
|
||||
}
|
||||
|
||||
}
|
||||
if(page>=100){
|
||||
stop = true;
|
||||
@@ -320,14 +328,13 @@ public class LianjiaCrawlByChrome {
|
||||
return detailUrls
|
||||
}
|
||||
|
||||
def parse(city,e, regionText, driver){
|
||||
def parse(city,e, regionText, driver, id){
|
||||
try {
|
||||
def price = e.select(".totalPrice .number").text()
|
||||
if(price){
|
||||
def detailUrl = e.select(".title a").attr("href")
|
||||
driver.get(detailUrl)
|
||||
def detailDoc = checkDocument(driver, detailUrl);
|
||||
|
||||
def title = e.select(".title a").text()
|
||||
if (title) {
|
||||
def detail = new BasicDBObject()
|
||||
@@ -345,6 +352,15 @@ public class LianjiaCrawlByChrome {
|
||||
if(detail.region){
|
||||
detail.region = RegionData.formatRegion(city, detail.region)
|
||||
}
|
||||
if(!detail.region){
|
||||
def str = detailDoc.toString().substring(detailDoc.toString().indexOf("resblockPosition")+1, detailDoc.toString().size()-1)
|
||||
str = str.substring(0, str.indexOf("',"))
|
||||
def locArrs = str.split(",")
|
||||
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0].toString().toDouble(), locArrs[1].toString().toDouble())
|
||||
if(gmap){
|
||||
detail.region = CommonDatautils.getRegionByLocation(city, gmap.lng+","+gmap.lat)
|
||||
}
|
||||
}
|
||||
|
||||
def roomStr = arrs[1]
|
||||
if (roomStr) {
|
||||
@@ -386,20 +402,22 @@ public class LianjiaCrawlByChrome {
|
||||
detail.source = "链家在线"
|
||||
detail.url = detailUrl
|
||||
|
||||
def liftStr = JQueryUtils.selectText("li:contains(梯户比例)",detailDoc).replace("梯户比例","")
|
||||
def detailDiv = JQueryUtils.select(".introContent", detailDoc)
|
||||
|
||||
def liftStr = detailDiv.select("li:contains(梯户比例)").text().replace("梯户比例","")
|
||||
if(liftStr){
|
||||
detail.lift = DataFormatUtils.parseLift(liftStr)
|
||||
detail.lift_household = DataFormatUtils.parseHousehold(liftStr)
|
||||
}
|
||||
detail.trade_property = JQueryUtils.selectText("li:contains(交易权属)", detailDoc).replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(户型结构)", detailDoc).replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房权所属)", detailDoc).replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(房屋年限)", detailDoc).replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(配备电梯)", detailDoc).replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑结构)", detailDoc).replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建成年代)", detailDoc).replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(建筑类型)", detailDoc).replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue(JQueryUtils.selectText("li:contains(所在楼层)", detailDoc).replace("所在楼层",""))
|
||||
detail.trade_property = detailDiv.select("li:contains(交易权属)").text().replace("交易权属","")
|
||||
detail.room_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(户型结构)").text().replace("户型结构",""))
|
||||
detail.property_ownership = DataFormatUtils.getValue( detailDiv.select("li:contains(房权所属)").text().replace("房权所属",""))
|
||||
detail.housing_age = DataFormatUtils.getValue( detailDiv.select("li:contains(房屋年限)").text().replace("房屋年限",""))
|
||||
detail.elevator = DataFormatUtils.getValue( detailDiv.select("li:contains(配备电梯)").text().replace("配备电梯",""))
|
||||
detail.build_structure = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑结构)").text().replace("建筑结构",""))
|
||||
detail.build_year = DataFormatUtils.getValue( detailDiv.select("li:contains(建成年代)").text().replace("建成年代",""))
|
||||
detail.build_type = DataFormatUtils.getValue( detailDiv.select("li:contains(建筑类型)").text().replace("建筑类型",""))
|
||||
def floorStr = DataFormatUtils.getValue( detailDiv.select("li:contains(所在楼层)").text().replace("所在楼层",""))
|
||||
|
||||
if(detail.total_price && detail.avg_price){
|
||||
def houseInfo = e.select(".houseInfo").text()
|
||||
@@ -417,7 +435,12 @@ public class LianjiaCrawlByChrome {
|
||||
detail.create_date = new Date()
|
||||
detail.m_date = new Date()
|
||||
println detail
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
if(id){
|
||||
DEAL_PRICE_COLL.update(new BasicDBObject("_id",id),new BasicDBObject('$set',detail))
|
||||
}else{
|
||||
DEAL_PRICE_COLL.save(detail)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -547,9 +570,9 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
// if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
if (!href.contains("fang.lianjia") && citys.contains(e.text())){
|
||||
cityList << [e.text(), href]
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
return cityList
|
||||
@@ -569,9 +592,11 @@ public class LianjiaCrawlByChrome {
|
||||
|
||||
WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。
|
||||
|
||||
elemUsername.sendKeys("15021384995");
|
||||
// elemUsername.sendKeys("15021384995");
|
||||
// elemPassword.sendKeys("12qwaszx");
|
||||
|
||||
elemPassword.sendKeys("12qwaszx");
|
||||
elemUsername.sendKeys("16283166371");
|
||||
elemPassword.sendKeys("Goojia123456");
|
||||
|
||||
btn.submit(); // 点击行为;
|
||||
Thread.sleep(15000)
|
||||
|
||||
+27
-7
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.poc
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -21,10 +22,11 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "C:\\Users\\gaojuan\\Desktop\\深圳政府指导价.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\指导价\\202306测试偏差较大案例统计数据——tob价格.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
|
||||
}
|
||||
|
||||
@@ -105,12 +107,12 @@ class AddPriceTest {
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['B']) {
|
||||
// def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
// if(dist){
|
||||
// def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['G'] = gxdPrice
|
||||
// }
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['I'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -129,4 +131,22 @@ class AddPriceTest {
|
||||
}
|
||||
return price
|
||||
}
|
||||
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-06-25","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
d['K'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+12
-3
@@ -28,9 +28,18 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
|
||||
|
||||
if(dists){
|
||||
for(def d : dists){
|
||||
def distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
|
||||
def distObj = null
|
||||
if(d.comm_id){
|
||||
distObj = CommonDatautils.findDistrictById(d.comm_id)
|
||||
}else{
|
||||
distObj = CommonDatautils.findDistrictByParams(d.city, d.region, d.name, "district", fields)
|
||||
}
|
||||
if(distObj && distObj._id){
|
||||
FzDataUtils.converDistrict(distObj, null, null);
|
||||
if(d.type && d.type==1){
|
||||
FzDataUtils.removeById(d.comm_id)
|
||||
}else{
|
||||
FzDataUtils.converDistrict(distObj, null, null);
|
||||
}
|
||||
println new Date().toString()+" 【处理完成】 "+d.city +" "+d.region+" "+d.name
|
||||
updateTaskStatus(d.id, 2);
|
||||
}
|
||||
@@ -57,7 +66,7 @@ class DayUpdateDistRefreshBatch extends BaseBatch{
|
||||
def q = new BasicDBObject()
|
||||
q.put("status", 5);//未处理
|
||||
TASK_COLL.find(q).each {d->
|
||||
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
|
||||
rets << ["id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name, "type":d.type, "comm_id":d.comm_id]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
+7
@@ -119,6 +119,13 @@ class FzDataUtils {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static def removeById(id){
|
||||
def del_data = new BasicDBObject()
|
||||
del_data.put("num_id", id)
|
||||
DIST_COLL.remove(del_data)
|
||||
}
|
||||
|
||||
static def saveDistrict(def m){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("num_id",m.num_id)
|
||||
|
||||
+4
@@ -294,4 +294,8 @@ class AddressUtil {
|
||||
return result
|
||||
}
|
||||
|
||||
public static void main(String[] args){
|
||||
def addr = AddressUtil.formatAddress("福州", "北三环路201号琴亭信和小区")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+46
-1
@@ -52,7 +52,40 @@ class InsidePolygon {
|
||||
|
||||
return list;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public static List<Point> createPoint3(String points){
|
||||
|
||||
List<Point> alllist = new ArrayList<Point>();
|
||||
if(points){
|
||||
List<String[]> arrs = new ArrayList<String[]>();
|
||||
if(points.contains("|")){
|
||||
String[] array = points.split("\\|");
|
||||
arrs.addAll(array)
|
||||
}else{
|
||||
arrs.add(points)
|
||||
}
|
||||
|
||||
for(def arrStr : arrs){
|
||||
List<Point> list = new ArrayList<Point>();
|
||||
String[] array = arrStr.split(";");
|
||||
for(int i =0 ;i<array.length-1;i++){
|
||||
def pointArrs = array[i].split("[,|\\|]")
|
||||
Point point = new Point();
|
||||
double x = Double.parseDouble(pointArrs[0]);
|
||||
double y = Double.parseDouble(pointArrs[1]);
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
alllist << list
|
||||
}
|
||||
}
|
||||
|
||||
return alllist;
|
||||
}
|
||||
|
||||
public static List<Map<String, Object>> getPoints(String points){
|
||||
String[] array = points.split("-");
|
||||
List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
|
||||
@@ -227,4 +260,16 @@ class InsidePolygon {
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public static def isInArea(lng, lat, pointList){
|
||||
def flag = false;
|
||||
for(def pointArr : pointList){
|
||||
flag = InsidePolygon.isPtInPoly(lng, lat, pointArr)
|
||||
if(flag){
|
||||
break;
|
||||
}
|
||||
}
|
||||
return flag;
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -24,7 +24,7 @@ class CommitteeLatLng {
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
|
||||
static void main(String[] args) {
|
||||
dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021))
|
||||
dealPoi(new BasicDBObject("location", "").append("year", 2022))
|
||||
//dealLocation()
|
||||
executorService.shutdown()
|
||||
}
|
||||
@@ -71,7 +71,7 @@ class CommitteeLatLng {
|
||||
if (amapPoi && amapPoi.size() > 0) {
|
||||
def poi = amapPoi[0]
|
||||
double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, name)
|
||||
if (similarity <= 0.8) {
|
||||
if (similarity <= 0.8 && !String.valueOf(poi.name).contains(name)) {
|
||||
amapPoi = GaoDeUtils.searchPoi(city, address, area, "")
|
||||
}
|
||||
poi = amapPoi[0]
|
||||
|
||||
+13
-9
@@ -55,13 +55,17 @@ class GaoDeUtils {
|
||||
* @return
|
||||
*/
|
||||
static String getGaodeKey(String type) {
|
||||
Map<String, String> params = new HashMap<>();
|
||||
params.put("type", type)
|
||||
JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params)
|
||||
if (json.status == 0 && json.result != null) {
|
||||
return json.result
|
||||
}
|
||||
System.exit(1);
|
||||
// Map<String, String> params = new HashMap<>();
|
||||
// params.put("type", type)
|
||||
// JSONObject json = HttpRequestUtil.getJsonContent("http://192.168.0.100:38080/FangJiaDataService/getAmapKey", params)
|
||||
// if (json.status == 0 && json.result != null) {
|
||||
// return json.result
|
||||
// }
|
||||
// System.exit(1);
|
||||
return "8796b9dfb3889b57c38e2fff1b7e51bd";
|
||||
// 'eb69e4ad2a9cdd31f163a0ed32effe6e', '60f764d228d64addf390e28d68a01a35', '66e6771767ab293ace441553c36621c1',
|
||||
// 'a63254b4a1b2e6f4bb11058e0b5d9fa2', '743ad872ad39f0a10665b28fa968af0b', '3b7c1d201c403bb416b63535ed3d0e98',
|
||||
// 'a34894d151fffff94f9a33bae3d5565c', '389e0a3a155e8d28728c3faede4ebd4f'
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -330,10 +334,10 @@ class GaoDeUtils {
|
||||
params.page = "${page}".toString()
|
||||
page++
|
||||
params.key = this.getGaodeKey("v5/place/text")
|
||||
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params);
|
||||
JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params);
|
||||
if (json == null) {
|
||||
params.key = this.getGaodeKey("v5/place/text")
|
||||
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params);
|
||||
json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params);
|
||||
}
|
||||
if (json && json.info && json.info == "OK") {
|
||||
count = json.getString("count").toInteger()
|
||||
|
||||
+40
@@ -1,7 +1,9 @@
|
||||
package com.fangjia.script.temp.shenjian.util
|
||||
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.HttpClient
|
||||
import org.apache.commons.httpclient.methods.PostMethod
|
||||
import org.apache.commons.httpclient.methods.RequestEntity
|
||||
import org.apache.commons.httpclient.methods.StringRequestEntity
|
||||
|
||||
/**
|
||||
@@ -26,4 +28,42 @@ class HttpUtils {
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
static def doPost2(String url, String jsonString){
|
||||
HttpClient httpClient = new HttpClient()
|
||||
PostMethod postMethod = new PostMethod(url)
|
||||
String responseMsg = null;
|
||||
try{
|
||||
StringRequestEntity requestEntity = new StringRequestEntity(
|
||||
jsonString,
|
||||
"application/json",
|
||||
"UTF-8");
|
||||
postMethod.setRequestEntity(requestEntity);
|
||||
httpClient.executeMethod(postMethod);
|
||||
InputStream is = postMethod.getResponseBodyAsStream();
|
||||
BufferedReader br = new BufferedReader(new InputStreamReader(is, "UTF-8"));
|
||||
StringBuffer sb = new StringBuffer("");
|
||||
String str= "";
|
||||
while((str = br.readLine()) != null){
|
||||
sb.append(str);
|
||||
}
|
||||
responseMsg = sb.toString();
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
} finally {
|
||||
postMethod.releaseConnection();
|
||||
}
|
||||
return JSONObject.fromObject(responseMsg);
|
||||
}
|
||||
|
||||
public String encodeHtml(String html) {
|
||||
String mh=html;
|
||||
try {
|
||||
mh=new String(html.getBytes("iso-8859-1"));
|
||||
} catch (UnsupportedEncodingException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return mh;
|
||||
}
|
||||
}
|
||||
|
||||
+5
-3
@@ -92,8 +92,10 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "¿ªÊ¼¼ÓÔØ¡¾${city}¡¿"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// if(city == "¹ãÖÝ"){
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias","estate_type2"])
|
||||
districtMap.putAll(detailMap)
|
||||
// }
|
||||
}
|
||||
|
||||
while(flag){
|
||||
@@ -833,7 +835,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
}
|
||||
}
|
||||
if(!dbo){
|
||||
if(!dbo && isStandardRoad(copyObj.address)){
|
||||
result = findAddrAliasFromMap(copyObj.city, copyObj.region, copyObj.address);
|
||||
if(result.size()==1){
|
||||
dbo = result[0];
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
}
|
||||
@Override
|
||||
void process(Object args) {
|
||||
println SHORT_DATE
|
||||
|
||||
def citys = CommonDatautils.findAllCity();
|
||||
|
||||
def weekend = DateUtils.getFirstDayOfWeek(new Date());
|
||||
|
||||
+19
-1
@@ -28,7 +28,25 @@ class TradeDealPriceStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeDealPriceStat()
|
||||
task.run()
|
||||
// task.run()
|
||||
task.deleteRepeat()
|
||||
}
|
||||
|
||||
def deleteRepeat(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source", "ÎÒ°®ÎÒ¼Ò")
|
||||
def cursor = DEAL_PRICE.find(q).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def set = new HashSet()
|
||||
for(def c : cursor){
|
||||
def tradeDate = DateUtils.formatDateByPattern(c.trade_date, "yyyyMMdd")
|
||||
def key = c.city +"#"+c.region+"#"+c.district_name+"#"+c.avg_price+"#"+tradeDate
|
||||
if(set.contains(key)){
|
||||
DEAL_PRICE.remove(new BasicDBObject('_id', c._id))
|
||||
println c._id
|
||||
}else{
|
||||
set << key
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
|
||||
+3
-3
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20230301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230401", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
@@ -38,6 +38,6 @@ while (startDate.before(endDate)) {
|
||||
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202303")
|
||||
ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301")
|
||||
|
||||
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20220901", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20221001", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20230201", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20230301", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.service.bo;
|
||||
|
||||
import java.io.Serializable;
|
||||
import java.util.Date;
|
||||
|
||||
/**
|
||||
* @author zhaojixiang
|
||||
* @since 2023/7/5
|
||||
*/
|
||||
public class BuildingRatioBo implements Serializable {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
private String id;
|
||||
private String city_name;
|
||||
private String region;
|
||||
private String district;
|
||||
private String building_name;
|
||||
private Double ratio;
|
||||
private Date c_date;
|
||||
private Date m_date;
|
||||
private String create_user;
|
||||
private String note;
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getCity_name() {
|
||||
return city_name;
|
||||
}
|
||||
|
||||
public void setCity_name(String city_name) {
|
||||
this.city_name = city_name;
|
||||
}
|
||||
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
|
||||
public void setRegion(String region) {
|
||||
this.region = region;
|
||||
}
|
||||
|
||||
public String getDistrict() {
|
||||
return district;
|
||||
}
|
||||
|
||||
public void setDistrict(String district) {
|
||||
this.district = district;
|
||||
}
|
||||
|
||||
public String getBuilding_name() {
|
||||
return building_name;
|
||||
}
|
||||
|
||||
public void setBuilding_name(String building_name) {
|
||||
this.building_name = building_name;
|
||||
}
|
||||
|
||||
public Double getRatio() {
|
||||
return ratio;
|
||||
}
|
||||
|
||||
public void setRatio(Double ratio) {
|
||||
this.ratio = ratio;
|
||||
}
|
||||
|
||||
public Date getC_date() {
|
||||
return c_date;
|
||||
}
|
||||
|
||||
public void setC_date(Date c_date) {
|
||||
this.c_date = c_date;
|
||||
}
|
||||
|
||||
public Date getM_date() {
|
||||
return m_date;
|
||||
}
|
||||
|
||||
public void setM_date(Date m_date) {
|
||||
this.m_date = m_date;
|
||||
}
|
||||
|
||||
public String getCreate_user() {
|
||||
return create_user;
|
||||
}
|
||||
|
||||
public void setCreate_user(String create_user) {
|
||||
this.create_user = create_user;
|
||||
}
|
||||
|
||||
public String getNote() {
|
||||
return note;
|
||||
}
|
||||
|
||||
public void setNote(String note) {
|
||||
this.note = note;
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
package com.fangjia.service.mongo.bkoff;
|
||||
|
||||
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author zhaojixiang
|
||||
* @since 2023/7/5
|
||||
*/
|
||||
public interface BuildingRatioService {
|
||||
public void save(BuildingRatio buildingRatio);
|
||||
public void deleteById(String id);
|
||||
public void saveOrUpdateByBuildingName(String city, String region, String district, String building_name, Map<String, Object> data);
|
||||
public BuildingRatio findBuildingRatioById(String id);
|
||||
public List<BuildingRatio> findAllByBuildingName(String city, String region, String district, String building_name);
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
package com.fangjia.service.mongo.bkoff;
|
||||
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.SortParam;
|
||||
import com.fangjia.dal.mongo.dao.bkoff.BuildingRatioDao;
|
||||
import com.fangjia.dal.mongo.model.bkoff.BuildingRatio;
|
||||
import com.fangjia.dal.sql.model.FjtUsers;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* @author zhaojixiang
|
||||
* @since 2023/7/5
|
||||
*/
|
||||
public class BuildingRatioServiceImpl implements BuildingRatioService{
|
||||
|
||||
private BuildingRatioDao buildingRatioDao;
|
||||
|
||||
@Override
|
||||
public void save(BuildingRatio buildingRatio) {
|
||||
|
||||
this.buildingRatioDao.save(buildingRatio);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void deleteById(String id) {
|
||||
this.buildingRatioDao.deleteById(id);
|
||||
}
|
||||
@Override
|
||||
public void saveOrUpdateByBuildingName(String city, String region, String district, String building_name, Map<String, Object> data) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("city", city));
|
||||
params.add(new QueryParam("region", region));
|
||||
params.add(new QueryParam("district", district));
|
||||
params.add(new QueryParam("building_name", building_name));
|
||||
data.put("city", city);
|
||||
data.put("region", region);
|
||||
data.put("district", district);
|
||||
data.put("building_name", building_name);
|
||||
this.buildingRatioDao.updateByParams(data, params, true);
|
||||
}
|
||||
|
||||
@Override
|
||||
public BuildingRatio findBuildingRatioById(String id) {
|
||||
|
||||
return this.buildingRatioDao.findById(id);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<BuildingRatio> findAllByBuildingName(String city, String region, String district, String building_name) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
if(StringUtils.isNotBlank(city)){
|
||||
params.add(new QueryParam("city", city));
|
||||
}
|
||||
if(StringUtils.isNotBlank(region)){
|
||||
params.add(new QueryParam("region", region));
|
||||
}
|
||||
if(StringUtils.isNotBlank(district)){
|
||||
params.add(new QueryParam("district", district));
|
||||
}
|
||||
if(StringUtils.isNotBlank(building_name)){
|
||||
params.add(new QueryParam("building_name", building_name));
|
||||
}
|
||||
|
||||
List<SortParam> sortParams = new ArrayList<SortParam>();
|
||||
sortParams.add(new SortParam("m_date", false));
|
||||
|
||||
return this.buildingRatioDao.findAllByParams(params, sortParams);
|
||||
}
|
||||
|
||||
public BuildingRatioDao getBuildingRatioDao() {
|
||||
return buildingRatioDao;
|
||||
}
|
||||
|
||||
public void setBuildingRatioDao(BuildingRatioDao buildingRatioDao) {
|
||||
this.buildingRatioDao = buildingRatioDao;
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,7 @@ public class ApiUserBo implements Serializable {
|
||||
private Date deadTime;
|
||||
private String manager;
|
||||
private String priceType;
|
||||
private Integer isMonitor;
|
||||
|
||||
public String getPriceType() {
|
||||
return priceType;
|
||||
@@ -162,4 +163,12 @@ public class ApiUserBo implements Serializable {
|
||||
public void setManager(String manager) {
|
||||
this.manager = manager;
|
||||
}
|
||||
|
||||
public Integer getIsMonitor() {
|
||||
return isMonitor;
|
||||
}
|
||||
|
||||
public void setIsMonitor(Integer isMonitor) {
|
||||
this.isMonitor = isMonitor;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user