This commit is contained in:
2024-06-19 13:51:11 +08:00
89 changed files with 8721 additions and 809 deletions
@@ -4,9 +4,13 @@ import com.fangjia.bkoff.model.DevelopmentZoneModel;
import com.fangjia.bkoff.model.JsonModel;
import com.fangjia.bkoff.model.LatlngPolygonModel;
import com.fangjia.core.util.GeoUtils;
import com.fangjia.core.util.ThreadLocalData;
import com.fangjia.core.util.ThreadLocalKey;
import com.fangjia.dal.mongo.model.DevelopmentZone;
import com.fangjia.dal.mongo.model.LatlngPolygon;
import com.fangjia.dal.mongo.model.StatsIndex;
import com.fangjia.dal.mongo.model.bkoff.BkoffUsers;
import com.fangjia.service.mongo.bkoff.BkoffUsersService;
import com.fangjia.service.mongo.fangjia.DevelopmentZoneService;
import com.fangjia.service.mongo.fangjia.LatlngPolygonService;
import org.apache.commons.lang.StringUtils;
@@ -22,13 +26,16 @@ import static com.fangjia.core.util.GeoUtils.LNG;
*/
public class DevelopmentZoneController {
private DevelopmentZoneService developmentZoneService;
private BkoffUsersService bkoffUsersService;
public Object listAction(DevelopmentZoneModel model) {
Pager pager = model.getPager(30);
if (pager.getCurrentPage() > 100) {
pager.setCurrentPage(1);
}
List<DevelopmentZone> developmentZones = developmentZoneService.findDevelopmentZone(model.getProvince(), model.getName(), pager);
long countDev = developmentZoneService.countDev(model.getProvince(),model.getName(), model.getLevel());
pager.setTotalRows((int) countDev);
List<DevelopmentZone> developmentZones = developmentZoneService.findDevelopmentZone(model.getProvince(),model.getName(), model.getLevel(), pager);
model.setDevelopmentZoneList(developmentZones);
model.setPager(pager);
return model;
@@ -62,44 +69,53 @@ public class DevelopmentZoneController {
}
public Object saveOrUpdateAction(DevelopmentZoneModel model) {
String userId = (String) ThreadLocalData.getInstance().get(ThreadLocalKey.UserId);
BkoffUsers user = bkoffUsersService.findBkoffUsersById(userId);
JsonModel ret = new JsonModel();
Map<String, Object> data = new HashMap<String, Object>();
ret.setJsonModel(data);
// name province bdLocation level 不能为空
if (StringUtils.isBlank(model.getProvince()) || StringUtils.isBlank(model.getName()) || model.getBdShape() == null || model.getBdShape().toString().equals("[]") || StringUtils.isBlank(model.getLevel())) {
data.put("msg", "保存失败");
return ret;
}
if (model.getId() == null) {
DevelopmentZone entity = new DevelopmentZone();
// name province bdLocation level 不能为空
if (StringUtils.isBlank(model.getProvince()) || StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getLevel())) {
data.put("msg", "保存失败(开发区名称,所在省份,级别不能为空)");
return ret;
}
DevelopmentZone entity = new DevelopmentZone();
// name province bdLocation level 不能为空/
entity.setName(model.getName());
entity.setProvince(model.getProvince());
String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", "");
List<List<Double>> bdLocationList = new ArrayList<List<Double>>();
String[] splitStr = locationStr.split(";");
for (int i = 0; i < splitStr.length; i++) {
String[] splitStr2 = splitStr[i].split(",");
List<Double> location = new ArrayList<Double>();
location.add(Double.parseDouble(splitStr2[0]));
location.add(Double.parseDouble(splitStr2[1]));
bdLocationList.add(location);
// 新加开发区,围栏不是必须有的
if (model.getBdShape() != null && !model.getBdShape().toString().equals("[]")) {
String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", "");
List<List<Double>> bdLocationList = new ArrayList<List<Double>>();
String[] splitStr = locationStr.split(";");
for (int i = 0; i < splitStr.length; i++) {
String[] splitStr2 = splitStr[i].split(",");
List<Double> location = new ArrayList<Double>();
location.add(Double.parseDouble(splitStr2[0]));
location.add(Double.parseDouble(splitStr2[1]));
bdLocationList.add(location);
}
//加上第一个,形成首尾闭合
bdLocationList.add(bdLocationList.get(0));
model.setBdShape(bdLocationList);
// 百度经纬度
Map<String, Object> bdShape = new HashMap<String, Object>();
bdShape.put("type", "polygon");
bdShape.put("coordinates", model.getBdShape());
entity.setBdShape(bdShape);
// 高德经纬度
Map<String, Object> shape = new HashMap<String, Object>();
shape.put("type", "polygon");
Object coordinates = convertGeoList(model.getBdShape());
shape.put("coordinates", coordinates);
entity.setShape(shape);
}
model.setBdShape(bdLocationList);
// 百度经纬度
Map<String, Object> bdShape = new HashMap<String, Object>();
bdShape.put("type", "polygon");
bdShape.put("coordinates", model.getBdShape());
entity.setBdShape(bdShape);
// 高德经纬度
Map<String, Object> shape = new HashMap<String, Object>();
shape.put("type", "polygon");
Object coordinates = convertGeoList(model.getBdShape());
shape.put("coordinates", coordinates);
entity.setShape(shape);
entity.setLevel(model.getLevel());
String aliasString = model.getAlias().replaceAll(",",",");
List<String> alias = Arrays.asList(aliasString.split("\\s*,\\s*"));
entity.setAlias(alias);
entity.setAddress(StringUtils.isBlank(model.getAddress()) ? "" : model.getAddress());
entity.setArea(StringUtils.isBlank(model.getArea()) ? "" : model.getArea());
entity.setBlock(StringUtils.isBlank(model.getBlock()) ? "" : model.getBlock());
@@ -109,50 +125,86 @@ public class DevelopmentZoneController {
entity.setNameEn(StringUtils.isBlank(model.getNameEn()) ? "" : model.getNameEn());
entity.setType(StringUtils.isBlank(model.getType()) ? "" : model.getType());
entity.setUrl(StringUtils.isBlank(model.getUrl()) ? "" : model.getUrl());
entity.setCityName(StringUtils.isBlank(model.getCityName()) ? "" : model.getCityName());
//记录操作人员
entity.setOperator(user.getRealName());
developmentZoneService.saveDevelopmentZone(entity);
data.put("msg", "保存成功");
} else {
DevelopmentZone entity = developmentZoneService.findDevelopmentZone(model.getId());
// Map<String, Object> udata = new HashMap<String, Object>();
// name province bdLocation level 不能为空
entity.setName(model.getName());
entity.setProvince(model.getProvince());
String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", "");
List<List<Double>> bdLocationList = new ArrayList<List<Double>>();
String[] splitStr = locationStr.split(";");
for (int i = 0; i < splitStr.length; i++) {
String[] splitStr2 = splitStr[i].split(",");
List<Double> location = new ArrayList<Double>();
location.add(Double.parseDouble(splitStr2[0]));
location.add(Double.parseDouble(splitStr2[1]));
bdLocationList.add(location);
// DevelopmentZone entity = developmentZoneService.findDevelopmentZone(model.getId());
Map<String, Object> udata = new HashMap<String, Object>();
if (StringUtils.isNotBlank(model.getProvince())) {
udata.put("province", model.getProvince());
}
model.setBdShape(bdLocationList);
// 百度经纬度
Map<String, Object> bdShape = new HashMap<String, Object>();
bdShape.put("type", "polygon");
bdShape.put("coordinates", model.getBdShape());
entity.setBdShape(bdShape);
// 高德经纬度
Map<String, Object> shape = new HashMap<String, Object>();
shape.put("type", "polygon");
Object coordinates = convertGeoList(model.getBdShape());
shape.put("coordinates", coordinates);
entity.setShape(shape);
entity.setLevel(model.getLevel());
entity.setAddress(StringUtils.isBlank(model.getAddress()) ? "" : model.getAddress());
entity.setArea(StringUtils.isBlank(model.getArea()) ? "" : model.getArea());
entity.setBlock(StringUtils.isBlank(model.getBlock()) ? "" : model.getBlock());
entity.setBuildDate(StringUtils.isBlank(model.getBuildDate()) ? "" : model.getBuildDate());
entity.setCreateTime(new Date());
entity.setIndustry(StringUtils.isBlank(model.getIndustry()) ? "" : model.getIndustry());
entity.setNameEn(StringUtils.isBlank(model.getNameEn()) ? "" : model.getNameEn());
entity.setType(StringUtils.isBlank(model.getType()) ? "" : model.getType());
entity.setUrl(StringUtils.isBlank(model.getUrl()) ? "" : model.getUrl());
developmentZoneService.updateDevelopmentZone(entity);
data.put("msg", "保存成功");
if (StringUtils.isNotBlank(model.getName())) {
udata.put("name", model.getName());
}
if (StringUtils.isNotBlank(model.getLevel())) {
udata.put("level", model.getLevel());
}
if (model.getBdShape() != null && !model.getBdShape().toString().equals("[]")) {
String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", "");
List<List<Double>> bdLocationList = new ArrayList<List<Double>>();
String[] splitStr = locationStr.split(";");
for (int i = 0; i < splitStr.length; i++) {
String[] splitStr2 = splitStr[i].split(",");
List<Double> location = new ArrayList<Double>();
location.add(Double.parseDouble(splitStr2[0]));
location.add(Double.parseDouble(splitStr2[1]));
bdLocationList.add(location);
}
//加上第一个,形成首尾闭合
bdLocationList.add(bdLocationList.get(0));
model.setBdShape(bdLocationList);
// 百度经纬度
Map<String, Object> bdShape = new HashMap<String, Object>();
bdShape.put("type", "polygon");
bdShape.put("coordinates", model.getBdShape());
udata.put("bdShape", bdShape);
// 高德经纬度
Map<String, Object> shape = new HashMap<String, Object>();
shape.put("type", "polygon");
Object coordinates = convertGeoList(model.getBdShape());
shape.put("coordinates", coordinates);
udata.put("shape", shape);
}
if (StringUtils.isNotBlank(model.getAlias()) ){
String aliasString = model.getAlias().replaceAll(",",",");
List<String> alias = Arrays.asList(aliasString.split("\\s*,\\s*"));
udata.put("alias", alias);
}
if (StringUtils.isNotBlank(model.getAddress())) {
udata.put("address", model.getAddress());
}
if (StringUtils.isNotBlank(model.getArea())) {
udata.put("area", model.getArea());
}
if (StringUtils.isNotBlank(model.getBlock())) {
udata.put("block", model.getBlock());
}
if (StringUtils.isNotBlank(model.getBuildDate())) {
udata.put("buildDate", model.getBuildDate());
}
if (StringUtils.isNotBlank(model.getIndustry())) {
udata.put("industry", model.getIndustry());
}
if (StringUtils.isNotBlank(model.getNameEn())) {
udata.put("nameEn", model.getNameEn());
}
if (StringUtils.isNotBlank(model.getType())) {
udata.put("type", model.getType());
}
if (StringUtils.isNotBlank(model.getUrl())) {
udata.put("url", model.getUrl());
}
if (StringUtils.isNotBlank(model.getCityName())) {
udata.put("cityName", model.getCityName());
}
udata.put("createTime", new Date());
//记录操作人员
udata.put("operator", user.getRealName());
developmentZoneService.updateDevelopmentZone(udata, model.getId());
data.put("msg", "修改成功");
}
return ret;
@@ -162,6 +214,10 @@ public class DevelopmentZoneController {
public void setDevelopmentZoneService(DevelopmentZoneService developmentZoneService) {
this.developmentZoneService = developmentZoneService;
}
public void setBkoffUsersService(BkoffUsersService bkoffUsersService) {
this.bkoffUsersService = bkoffUsersService;
}
//将百度坐标转为高德坐标
public List<List<Double>> convertGeoList(List<List<Double>> bdLocationList) {
@@ -17,20 +17,38 @@ public class DevelopmentZoneModel extends BaseModel {
private String level;
private String name;
private String alias;
// private List<String> addr_alias;
private String nameEn;
private String province;
private String type;
private String url;
private String district;
List<List<Double>> shape;
List<List<Double>> bdShape;
private String block;
private DevelopmentZone developmentZone;
private List<DevelopmentZone> developmentZoneList;
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
public String getId() {
return id;
}
@@ -9,7 +9,7 @@
<div class="city-border">
<div class="city-border-left hide" id="city-border-left"></div>
<div id="city-border-mid" class="">
<a title="选择城市" class="city-current" id="wt-city-title"><#if defaultValue?? && defaultValue != ''>${cityDisplayMap[defaultValue]}<#elseif showClear>无<#elseif showAll>全部<#else>选择城市</#if></a>
<a title="选择城市" class="city-current" id="wt-city-title"><#if defaultValue?? && defaultValue != '' &&cityDisplayMap[defaultValue]??>${cityDisplayMap[defaultValue]}<#elseif showClear>无<#elseif showAll>全部<#else>选择城市</#if></a>
<span class="downtip" id="wt-city-other"></span>
</div>
<div class="city-border-right hide" id="city-border-right"></div>
@@ -3,147 +3,216 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/html">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>开发区围栏列表</title>
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>开发区围栏列表</title>
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
</head>
<body>
<#include "/common_js.ftl" />
<#include "/layout/header.ftl" />
<div class="main-content">
<div class="menuButton">
<a href="../developmentZone/list" class="list">开发区列表</a>&nbsp;&nbsp;
</div>
<#include "/common_js.ftl" />
<#include "/layout/header.ftl" />
<div class="main-content">
<div class="menuButton">
<a href="../developmentZone/list" class="list">开发区列表</a>&nbsp;&nbsp;
</div>
<div>
<form id="form1">
<#if developmentZone??>
<input id="id" name="id" type="hidden" value="${developmentZone.id!}"/>
</#if>
<div>
<form id="form1">
<#if developmentZone??>
<input id="id" name="id" type="hidden" value="${developmentZone.id!}"/>
</#if>
<table>
<tr>
<td>
<#if latlngPolygon??>
<@d_city defaultValue="${latlngPolygon.city!}" callback="changeCity" callbackField="display" name="cityDisplay" />
<#else>
<@d_city defaultValue="上海" callback="changeCity" callbackField="display" name="cityDisplay" />
</#if>
</td>
<td>
<label for="name">开发区名称:</label> <input id="name" name="name" type="text" value="<#if developmentZone??>${developmentZone.name!}</#if>"/>
<label for="nameEn">开发区名称拼音:</label> <input id="nameEn" name="nameEn" type="text" value="<#if developmentZone??>${developmentZone.nameEn!}</#if>"/>
<label for="province">所在省份:</label> <input id="province" name="province" type="text" value="<#if developmentZone??>${developmentZone.province!}</#if>"/>
<label for="level">级别:</label> <input id="level" name="level" type="text" value="<#if developmentZone??>${developmentZone.level!}</#if>"/>
<label for="address">地址:</label> <input id="address" name="address" type="text" value="<#if developmentZone??>${developmentZone.address!}</#if>"/>
<label for="area">所在地区:</label> <input id="area" name="area" type="text" value="<#if developmentZone??>${developmentZone.area!}</#if>"/>
<label for="buildDate">建立年代:</label><input id="buildDate" name="buildDate" type="text" value="<#if developmentZone??>${developmentZone.buildDate!}</#if>"/>
<label for="type">类型:</label><input id="type" name="type" type="text" value="<#if developmentZone??>${developmentZone.type!}</#if>"/>
<label for="url">URL:</label><input id="url" name="url" type="text" value="<#if developmentZone??>${developmentZone.url!}</#if>"/>
<label for="industry">主导产业:</label><input id="industry" name="industry" type="text" value="<#if developmentZone??>${developmentZone.industry!}</#if>"/>
<label for="block">区块:</label><input id="block" name="block" type="text" value="<#if developmentZone??>${developmentZone.block!}</#if>"/>
<input id="saveOrUpdate" type="button" value="提交"/>
</td>
</tr>
<tr>
<td colspan="2">
<input name="bdShape" id="bdShape" type="hidden"/>
<div id="allmap" style="width:100%;height:900px;"></div>
</td>
</tr>
</table>
</form>
</div>
</div>
<table>
<tr>
<td>
<#if developmentZone??>
<@d_city defaultValue="${developmentZone.cityName!}" callback="changeCity" callbackField="display" name="cityDisplay" />
<#else>
<@d_city defaultValue="上海" callback="changeCity" callbackField="display" name="cityDisplay" />
</#if>
</td>
<td>
<label for="name">开发区名称:</label> <input id="name" name="name" type="text" placeholder="必填" value="<#if developmentZone??>${developmentZone.name!}</#if>"/>
<label for="alias">别名:</label>
<input id="alias" name="alias" type="text" placeholder="多个别名用,隔开" value="<#if developmentZone??><#list (developmentZone.alias)! as alia>${alia + ','}</#list><#else></#if>"/>
<label for="nameEn">开发区名称拼音:</label> <input id="nameEn" name="nameEn" type="text" value="<#if developmentZone??>${developmentZone.nameEn!}</#if>"/>
<label for="province">所在省份:</label> <input id="province" name="province" type="text" placeholder="必填" value="<#if developmentZone??>${developmentZone.province!}</#if>"/><br>
<label for="cityName">城市:</label><input id="cityName" name="cityName" type="text" value="<#if developmentZone??>${developmentZone.cityName!}</#if>"/>
<label for="district">区域:</label><input id="district" name="district" type="text" value="<#if developmentZone??>${developmentZone.district!}</#if>"/>
<label for="level">级别:</label> <input id="level" name="level" type="text" placeholder="必填" value="<#if developmentZone??>${developmentZone.level!}</#if>"/>
<label for="address">地址:</label> <input id="address" name="address" type="text" value="<#if developmentZone??>${developmentZone.address!}</#if>"/><br>
<label for="area">所在地区:</label> <input id="area" name="area" type="text" value="<#if developmentZone??>${developmentZone.area!}</#if>"/>
<label for="buildDate">建立年代:</label><input id="buildDate" name="buildDate" type="text" value="<#if developmentZone??>${developmentZone.buildDate!}</#if>"/>
<label for="type">类型:</label><input id="type" name="type" type="text" value="<#if developmentZone??>${developmentZone.type!}</#if>"/>
<label for="url">URL:</label><input id="url" name="url" type="text" value="<#if developmentZone??>${developmentZone.url!}</#if>"/><br>
<label for="industry">主导产业:</label><input id="industry" name="industry" type="text" value="<#if developmentZone??>${developmentZone.industry!}</#if>"/>
<label for="block">区块:</label><input id="block" name="block" type="text" value="<#if developmentZone??>${developmentZone.block!}</#if>"/><br/>
<input id="saveOrUpdate" type="button" value="提交"/>
</td>
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<!--加载鼠标绘制工具-->
<script type="text/javascript" src="http://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"></script>
<link rel="stylesheet" href="http://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.css" />
<!--加载检索信息窗口-->
<script type="text/javascript" src="http://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"></script>
<link rel="stylesheet" href="http://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.css" />
</tr>
<tr>
<td colspan="2">
<input name="bdShape" id="bdShape" type="hidden"/>
<div id="allmap" style="width:100%;height:900px;"></div>
</td>
</tr>
</table>
</form>
</div>
</div>
<script type="application/javascript">
var map = new BMap.Map("allmap"); // 创建Map实例
if($("#cityDisplay").val()) {
map.centerAndZoom($("#cityDisplay").val(), 13);
}else {
map.centerAndZoom('上海', 13);
}
map.enableScrollWheelZoom(true);
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<!--加载鼠标绘制工具-->
<script type="text/javascript" src="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"></script>
<link rel="stylesheet" href="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.css" />
<!--加载检索信息窗口-->
<script type="text/javascript" src="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"></script>
<link rel="stylesheet" href="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.css" />
function changeCity(display){
clearAll();
map.centerAndZoom(display, 13);
}
<script type="application/javascript">
var map = new BMap.Map("allmap"); // 创建Map实例
if($("#cityDisplay").val()) {
map.centerAndZoom($("#cityDisplay").val(), 13);
}else {
map.centerAndZoom('上海', 13);
}
map.enableScrollWheelZoom(true);
var overlays = [];
var overlaycomplete = function(e){
overlays.push(e.overlay);
var points = e.overlay.getPath();
function changeCity(display){
clearAll();
map.centerAndZoom(display, 13);
}
var bdShape = [];
for(var i=0;i<points.length;i++) {
var location = [];
location.push(points[i].lng );
location.push(points[i].lat );
bdShape.push(location);
}
$("#bdShape").val(bdShape.join(";"));
};
var overlays = [];
var overlaycomplete = function(e){
overlays.push(e.overlay);
var points = e.overlay.getPath();
var styleOptions = {
strokeColor:"red", //边线颜色。
fillColor:"red", //填充颜色。当参数为空时,圆形将没有填充效果。
strokeWeight: 3, //边线的宽度,以像素为单位。
strokeOpacity: 0.8, //边线透明度,取值范围0 - 1。
fillOpacity: 0.6, //填充的透明度,取值范围0 - 1。
strokeStyle: 'solid' //边线的样式,solid或dashed。
var bdShape = [];
for(var i=0;i<points.length;i++) {
var location = [];
location.push(points[i].lng );
location.push(points[i].lat );
bdShape.push(location);
}
//实例化鼠标绘制工具
var drawingManager = new BMapLib.DrawingManager(map, {
isOpen: false, //是否开启绘制模式
enableDrawingTool: true, //是否显示工具栏
drawingToolOptions: {
anchor: BMAP_ANCHOR_TOP_RIGHT, //位置
offset: new BMap.Size(5, 5), //偏离值
},
circleOptions: styleOptions, //圆的样式
polylineOptions: styleOptions, //线的样式
polygonOptions: styleOptions, //多边形的样式
rectangleOptions: styleOptions //矩形的样式
});
//添加鼠标绘制工具监听事件,用于获取绘制结果
drawingManager.addEventListener('overlaycomplete', overlaycomplete);
function clearAll() {
for(var i = 0; i < overlays.length; i++){
map.removeOverlay(overlays[i]);
$("#bdShape").val(bdShape.join(";"));
// 在创建多边形后立即启用编辑
e.overlay.enableEditing();
};
var styleOptions = {
strokeColor:"red", //边线颜色。
fillColor:"red", //填充颜色。当参数为空时,圆形将没有填充效果。
strokeWeight: 1, //边线的宽度,以像素为单位。
strokeOpacity: 0.8, //边线透明度,取值范围0 - 1。
fillOpacity: 0.6, //填充的透明度,取值范围0 - 1。
strokeStyle: 'solid' //边线的样式,solid或dashed。
}
//实例化鼠标绘制工具
var drawingManager = new BMapLib.DrawingManager(map, {
isOpen: true, //是否开启绘制模式
enableDrawingTool: true, //是否显示工具栏
drawingToolOptions: {
anchor: BMAP_ANCHOR_TOP_RIGHT, //位置
offset: new BMap.Size(5, 5), //偏离值
},
circleOptions: styleOptions, //圆的样式
polylineOptions: styleOptions, //线的样式
polygonOptions: styleOptions, //多边形的样式
rectangleOptions: styleOptions //矩形的样式
});
//添加鼠标绘制工具监听事件,用于获取绘制结果
drawingManager.addEventListener('overlaycomplete', overlaycomplete);
function clearAll() {
for(var i = 0; i < overlays.length; i++){
map.removeOverlay(overlays[i]);
}
overlays.length = 0;
}
// 添加点击覆盖物事件监听器
map.addEventListener("click", function(e) {
// 判断点击的是否是已存在的覆盖物
if (e.overlay) {
// 如果是多边形或折线,则可以编辑
if (e.overlay instanceof BMap.Polygon || e.overlay instanceof BMap.Polyline) {
// 开启覆盖物编辑功能
e.overlay.enableEditing();
}
// 如果是标记点,则可以拖动
else if (e.overlay instanceof BMap.Marker) {
// 开启标记点拖动功能
e.overlay.enableDragging();
}
overlays.length = 0;
}
});
$("#saveOrUpdate").click(function(){
jQuery.ajax({
type: 'POST',
url: '../developmentZone/saveOrUpdate',
data: $("#form1").serialize(),
dataType: 'json',
timeout: 100000,
success: function(data) {
alert(data.msg);
location.href="list";
},
error : function() {
// 添加覆盖物编辑完成事件监听器
map.addEventListener("overlaycomplete", function(e) {
// 判断新增的覆盖物是否是多边形或折线
if (e.overlay instanceof BMap.Polygon || e.overlay instanceof BMap.Polyline) {
// 添加覆盖物编辑完成事件监听器
e.overlay.addEventListener("lineupdate", function() {
// 编辑完成后获取更新后的路径点,并更新表单中的数据
var points = e.overlay.getPath();
var bdShape = [];
for (var i = 0; i < points.length; i++) {
var location = [];
location.push(points[i].lng);
location.push(points[i].lat);
bdShape.push(location);
}
$("#bdShape").val(bdShape.join(";"));
});
}
});
})
$("#saveOrUpdate").click(function(){
// 更新表单中的点位置数据
updatePolygonPoints();
jQuery.ajax({
type: 'POST',
url: '../developmentZone/saveOrUpdate',
data: $("#form1").serialize(),
dataType: 'json',
timeout: 100000,
success: function(data) {
alert(data.msg);
location.href="list";
},
error : function() {
}
});
});
// 更新表单中的多边形点位置数据
function updatePolygonPoints() {
var polygons = map.getOverlays(BMap.Polygon);
for (var i = 0; i < polygons.length; i++) {
// 确保当前对象是一个多边形对象
if (polygons[i] instanceof BMap.Polygon) {
var points = polygons[i].getPath();
var bdShape = [];
for (var j = 0; j < points.length; j++) {
var location = [];
location.push(points[j].lng);
location.push(points[j].lat);
bdShape.push(location);
}
// 将更新后的点位置数据更新到表单中
$("#bdShape").val(bdShape.join(";"));
}
}
}
</script>
</script>
</body>
</html>
</#compress>
</#compress>
@@ -14,7 +14,7 @@
<#include "/layout/header.ftl" />
<div class="main-content">
<div class="menuButton">
<a href="../developmentZone/edit" target="_blank" class="list">新增开发区围栏</a>&nbsp;&nbsp;
<a href="../developmentZone/edit" target="_blank" class="list">新增开发区</a>&nbsp;&nbsp;
</div>
<form action="../developmentZone/list" id="_search_log_form">
<div>
@@ -24,32 +24,58 @@
<label for="name">名称:</label>
<input type="text" name="name" id="name" class="input-medium" style="margin-top:9px;"
value="${name!}"></input>
<label for="name">开发区级别:</label>
<input type="text" name="level" id="level" class="input-medium" style="margin-top:9px;"
value="${level!}"></input>
<input id="queryBtn" class="btn btn-primary" type="submit" value="查询"/>
</div>
</form>
<div class="list" style="margin-top:5px;">
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
<table style="float:left;" name="reginTable" class="regionTab">
<tr>
<th>名称</th>
<th>级别</th>
<th>别名</th>
<th>开发区级别</th>
<th>省份</th>
<th style="width: 160px">建立时间</th>
<th>城市</th>
<th>区域</th>
<th>区块</th>
<th>是否有围栏</th>
<th>操作人员</th>
<th>修改日期</th>
<th style="width: 120px;">操作</th>
</tr>
<#list developmentZoneList as it>
<tr>
<td>${it.name!}</td>
<td>
<#if it.alias??>
<#list it.alias as item>
${item?string}<#if item_has_next>, </#if>
</#list>
</#if>
</td>
<td>${it.level!}</td>
<td>${it.province!}</td>
<td>${it.cityName!}</td>
<td>${it.district!}</td>
<td>${it.block!}</td>
<td>
${it.buildDate}
<#if it.bdShape?has_content>
已有围栏
</#if>
</td>
<td>${it.operator!}</td>
<td>${it.createTime!?string("yyyy-MM-dd HH:mm:ss")}</td>
<td>
<a href="show?id=${it.id!}" target="_blank">查看</a>
<a href="../developmentZone/edit?id=${it.id!}" target="_blank">新建围栏</a>
<a name="del${it_index}" href="javascript:;" ref="delete?id=${it.id!}">删除</a>
<#-- <a name="del${it_index}" href="javascript:;" ref="delete?id=${it.id!}">删除</a>-->
</td>
</tr>
@@ -31,6 +31,14 @@
<td>省份</td>
<td>${developmentZone.province!}</td>
</tr>
<tr>
<td>城市</td>
<td>${developmentZone.cityName!}</td>
</tr>
<tr>
<td>区域</td>
<td>${developmentZone.district!}</td>
</tr>
<tr>
<td>级别</td>
<td>${developmentZone.level!}</td>
@@ -39,6 +47,16 @@
<td>名称</td>
<td>${developmentZone.name!}</td>
</tr>
<tr>
<td>别名</td>
<td>
<#if developmentZone.alias??>
<#list developmentZone.alias as item>
${item?string}<#if item_has_next>, </#if>
</#list>
</#if>
</td>
</tr>
<tr>
<td>区块</td>
<td>${developmentZone.block!}</td>
@@ -48,21 +66,22 @@
<div id="allmap" style="width:1024px;height:600px;"></div>
</td>
</tr>
<input id="saveOrUpdate" type="button" value="提交"/>
</table>
</form>
</div>
</div>
<script type="text/javascript"
src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<!--加载鼠标绘制工具-->
<script type="text/javascript"
src="http://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"></script>
<link rel="stylesheet" href="http://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.css"/>
src="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"></script>
<link rel="stylesheet" href="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.css"/>
<!--加载检索信息窗口-->
<script type="text/javascript"
src="http://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"></script>
<link rel="stylesheet" href="http://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.css"/>
src="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"></script>
<link rel="stylesheet" href="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.css"/>
<script type="application/javascript">
var map = new BMap.Map("allmap"); // 创建Map实例
@@ -71,7 +90,7 @@
<#if developmentZone.bdShape??>
var polygon = new BMap.Polygon([
<#list developmentZone.bdShape.coordinates as latlng>
new BMap.Point(${latlng[0]}, ${latlng[1]})<#if latlng_index lt developmentZone.bdShape.coordinates?size>, </#if>
new BMap.Point(${latlng[0]?string("#.######")}, ${latlng[1]?string("#.######")})<#if latlng_index lt developmentZone.bdShape.coordinates?size>, </#if>
</#list>
], {strokeColor: "blue", strokeWeight: 2, strokeOpacity: 0.5});
</#if>
@@ -79,6 +98,48 @@
map.centerAndZoom(polygon.getBounds().getCenter(), 13);
map.addOverlay(polygon); //增加多边形
// 添加可编辑功能
polygon.enableEditing();
$("#saveOrUpdate").click(function(){
// 更新表单中的点位置数据
updatePolygonPoints();
jQuery.ajax({
type: 'POST',
url: '../developmentZone/saveOrUpdate',
data: $("#form1").serialize(),
dataType: 'json',
timeout: 100000,
success: function(data) {
alert(data.msg);
location.href="list";
},
error : function() {
}
});
});
// 更新表单中的多边形点位置数据
function updatePolygonPoints() {
var polygons = map.getOverlays(BMap.Polygon);
for (var i = 0; i < polygons.length; i++) {
// 确保当前对象是一个多边形对象
if (polygons[i] instanceof BMap.Polygon) {
var points = polygons[i].getPath();
var bdShape = [];
for (var j = 0; j < points.length; j++) {
var location = [];
location.push(points[j].lng);
location.push(points[j].lat);
bdShape.push(location);
}
// 将更新后的点位置数据更新到表单中
$("#bdShape").val(bdShape.join(";"));
}
}
}
</script>
@@ -1052,8 +1052,8 @@
<script type="text/javascript" src="../js/kindeditor-4.0/lang/zh_CN.js"></script>
<script type="text/javascript" src="../js/numValidator.js"></script>
<#--<script language="javascript" src="http://app.mapabc.com/apis?t=javascriptmap&v=3.1.1&key=b0a7db0b3a30f944a21c3682064dc70ef5b738b062f6479a5eca39725798b1ee300bd8d5de3a4ae3"></script>-->
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="http://webapi.amap.com/maps?v=1.4.3&key=65d0404c8a96fc2a630d9621c10c25bf"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://webapi.amap.com/maps?v=1.4.3&key=65d0404c8a96fc2a630d9621c10c25bf"></script>
<script type="text/javascript">
var flag=false;
$(document).ready(function(){
@@ -49,6 +49,7 @@
<@menuitem uri='/agentMaster/listMaster' text="中介公司" />
<@menuitem uri='/mortgageInfo/list' text="抵押品" />
<@menuitem uri='/developmentZone/list' text="开发区" />
<@menuitem uri='/township/list' text="乡镇街道" />
</ul>
</li>
<li class="menu">
@@ -929,7 +929,7 @@
</div><!-- /container -->
<div style="bottom: 1px;left: 45%;position: fixed;"><button type="button" id="editDist" class="btn btn-default btn-info form-control" style="width:200px;"> 保 存 </button></div>
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript">
$("#region").change(function(){
@@ -12,7 +12,7 @@
</style>
<#include "/common_js.ftl" />
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript">
$(document).ready(function(){
$("#queryBtn").click(function(){
@@ -12,7 +12,7 @@
</style>
<#include "/common_js.ftl" />
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript">
$(document).ready(function(){
$("#queryBtn").click(function(){
@@ -0,0 +1,16 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.MongoDao;
import com.fangjia.dal.mongo.model.AliasFang;
import java.util.List;
import java.util.Set;
/**
* @author shenjian
* @since 2024/6/11
*/
public interface AliasFangDao extends MongoDao<AliasFang> {
List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names);
}
@@ -0,0 +1,40 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.mongo.model.AliasFang;
import com.fangjia.dal.mongo.model.ConsumerAddress;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
import org.apache.commons.lang.StringUtils;
import org.sothis.core.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
public class AliasFangDaoImpl extends MongoDaoImpl<AliasFang> implements AliasFangDao{
@Override
public List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names) {
DBObject query = new BasicDBObject();
if (StringUtils.isNotBlank(city)) {
query.put("city", city);
}
if (StringUtils.isNotBlank(region)) {
query.put("region", region);
}
// db.alias_fang.find({'city': 'ÖØÇì', 'region': 'Óå±±', $or: [{'name': 'ÖйúĦ'}, {'alias': 'ÖйúĦ'}]})
if(CollectionUtils.isNotEmpty(names)) {
BasicDBList dbList = new BasicDBList();
dbList.add(new BasicDBObject("name", new BasicDBObject("$in", names)));
dbList.add(new BasicDBObject("alias", new BasicDBObject("$in", names)));
query.put("$or", dbList);
}
List<AliasFang> objs = find(query, null, null, 0, Integer.MAX_VALUE);
return objs;
}
}
@@ -0,0 +1,68 @@
package com.fangjia.dal.mongo.model;
import com.fangjia.dal.MongoEntity;
import javax.persistence.*;
/**
* ËÑ·¿Ãû³ÆÓë±ðÃû¹ØÏµ
*/
@Entity
@Table(name = "business.alias_fang")
public class AliasFang implements MongoEntity {
private String id;
private String city;
private String region;
private String name;
private String alias;
@Override
@Id
@GeneratedValue
@Column(name = "_id", unique = true, nullable = false)
public String getId() {
return id;
}
@Override
public void setId(String id) {
this.id = id;
}
@Column(name = "city")
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
@Column(name = "region")
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
@Column(name = "name")
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Column(name = "alias")
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
}
@@ -31,6 +31,12 @@ public class DevelopmentZone implements MongoEntity {
private Map<String, Object> bdShape;
private String block;
private String operator;
private String cityName;
private String district;
//增加别名字段
private List<String> alias;
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@@ -170,4 +176,40 @@ public class DevelopmentZone implements MongoEntity {
public void setBlock(String block) {
this.block = block;
}
@Column(name = "operator")
public String getOperator() {
return operator;
}
public void setOperator(String operator) {
this.operator = operator;
}
@Column(name = "city")
public String getCityName() {
return cityName;
}
public void setCityName(String cityName) {
this.cityName = cityName;
}
@Column(name = "district")
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
@Column(name = "alias")
public List<String> getAlias() {
return alias;
}
public void setAlias(List<String> alias) {
this.alias = alias;
}
}
@@ -187,7 +187,7 @@ public class District implements SeaweedEntity {
private double score;
// 小区性质
private String landProperty;
private String masterDistrictNname;
private String masterDistrictName;
private String ip;
private String addrType;
private boolean selected;
@@ -309,12 +309,12 @@ public class District implements SeaweedEntity {
}
@Column(name = "master_district_name")
public String getMasterDistrictNname() {
return masterDistrictNname;
public String getMasterDistrictName() {
return masterDistrictName;
}
public void setMasterDistrictNname(String masterDistrictNname) {
this.masterDistrictNname = masterDistrictNname;
public void setMasterDistrictName(String masterDistrictName) {
this.masterDistrictName = masterDistrictName;
}
public void setCategory(String category) {
@@ -7,7 +7,7 @@
<link href="../style/bootstrap/css/bootstrap.min.css" rel="stylesheet" media="screen">
<link href="//cdn.bootcss.com/bootstrap/3.3.5/css/bootstrap-theme.min.css" rel="stylesheet" id="bs-theme-stylesheet">
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
</head>
<body>
<div>
@@ -7,7 +7,7 @@
<link href="../style/bootstrap/css/bootstrap.min.css" rel="stylesheet" media="screen">
<link href="//cdn.bootcss.com/bootstrap/3.3.5/css/bootstrap-theme.min.css" rel="stylesheet" id="bs-theme-stylesheet">
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
</head>
<body>
<div>
@@ -14,11 +14,31 @@ import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.service.bo.DistrictStatBo;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.district.DistrictStatService;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import sun.misc.BASE64Encoder;
import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
import java.util.Stack;
/**
* 地址治理
@@ -358,8 +378,115 @@ public class AddrSvcController {
return mav;
}
private static final Map<String, String> areaMap = new HashMap<String, String>() {
{
// 添加键值对到 Map 中
put("111", "主城区");
put("112", "城乡结合区");
put("121", "镇中心区");
put("122", "镇乡结合区");
put("123", "特殊区域");
put("210", "乡中心区");
put("220", "村庄");
}
};
/**
* 涉农识别-新规则
* 涉农识别-新规则-南京银行
*
* @param model
* @return
*/
public Object njcbVillageIdentifyAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
String address = model.getAddress();
params.put("address", address);
String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null;
params.put("type", type);
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8");
} catch (Exception e) {
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8");
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("msgCode");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
//我们的结果
JSONObject result = json.getJSONObject("result");
//我们省级及以上有开发区,直接用我们的结果
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) {
//结果比较,如果涉农结果不一致
//调用齐梁 type :1 对公 0 对私
//我们 type :1 对私 2 对共
String qltype = "2".equals(type) ? "1" : "0";
Map<String, String> map = senRequest(address, qltype);
// 涉农结果不一样
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
//城市不一样 用我们
if (result.containsKey("city")) {
if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
basicModel.setResult(result);
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
basicModel.setResult(result);
}
} else {
basicModel.setResult(result);
}
} catch (Exception e) {
e.printStackTrace();
basicModel.setResult(result);
}
}
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 涉农识别-新规则-基础版本
*
* @param model
* @return
@@ -372,6 +499,207 @@ public class AddrSvcController {
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
String address = model.getAddress();
params.put("address", address);
String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null;
params.put("type", type);
params.put("showFullValue", "2");
// params.put("showTestValue", "2");
Integer radius = model.getRadius();
params.put("radius", (radius != null && radius != 0) ? radius.toString() : null);
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
} catch (Exception e) {
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("msgCode");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
//我们的结果
JSONObject result = json.getJSONObject("result");
//我们省级及以上有开发区,直接用我们的结果
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) {
//结果比较,如果涉农结果不一致
//调用齐梁 type :1 对公 0 对私
//我们 type :1 对私 2 对共
String qltype = "2".equals(type) ? "1" : "0";
Map<String, String> map = senRequest(address, qltype);
// 涉农结果不一样
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
//有市辖建制镇, 或者城市不一样 用我们
if (result.containsKey("city")) {
if ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
basicModel.setResult(result);
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
basicModel.setResult(result);
}
} else {
basicModel.setResult(result);
}
} catch (Exception e) {
e.printStackTrace();
basicModel.setResult(result);
}
}
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
private static Map<String, String> senRequest(String address, String type) throws IOException {
String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
String proxyHost = "tunnel2.qg.net";
int proxyPort = 14186;
final String proxyUsername = "113BBC06";
final String proxyPassword = "3B2FBD05B0AF";
String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}";
String authKey = proxyUsername;
String password = proxyPassword;
String auth = authKey + ":" + password;
String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8")));
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(targetURL);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
Authenticator.setDefault(new Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray());
}
});
httpPost.setHeader("Content-Type", "application/json");
httpPost.setHeader("Accept", "*/*");
httpPost.setHeader("Host", "146.56.217.44:9010");
httpPost.setHeader("Proxy-Authorization", authHeader);
StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON);
httpPost.setEntity(stringEntity);
CloseableHttpResponse httpResponse = null;
int retryCount = 2; // 设置重试次数
int retryDelay = 10; // 设置重试延迟时间(毫秒)
boolean success = false;
for (int i = 0; i < retryCount; i++) {
try {
httpResponse = httpClient.execute(httpPost);
success = true;
break;
} catch (IOException e) {
System.err.println("代理请求失败,进行重试...");
try {
Thread.sleep(retryDelay);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
}
if (success) {
HttpEntity responseEntity = httpResponse.getEntity();
String responseString = EntityUtils.toString(responseEntity);
// 使用 Gson 解析 JSON 字符串
Gson gson = new Gson();
JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class);
// 创建 Map 存储各个值
Map<String, String> dataMap = new HashMap<String, String>();
// 获取各个值并放入 Map
// dataMap.put("code", jsonObject.get("code").getAsString());
// dataMap.put("msg", jsonObject.get("msg").getAsString());
JsonElement dataElement = jsonObject.get("data");
if (dataElement != null && !dataElement.isJsonNull() && dataElement.isJsonObject()) {
JsonObject data = dataElement.getAsJsonObject();
// 在这里继续处理 data 对象
if (data != null && !data.isJsonNull()) {
// dataMap.put("version", data.get("version").getAsString());
dataMap.put("province", data.get("province").getAsString());
dataMap.put("city", data.get("city").getAsString());
dataMap.put("district", data.get("district").getAsString());
dataMap.put("town", data.get("town").getAsString());
dataMap.put("committee", data.get("village").getAsString());
dataMap.put("provinceCode", data.get("provinceCode").getAsString());
dataMap.put("cityCode", data.get("cityCode").getAsString());
dataMap.put("districtCode", data.get("districtCode").getAsString());
dataMap.put("townCode", data.get("townCode").getAsString());
dataMap.put("committeeCode", data.get("villageCode").getAsString());
dataMap.put("typeCode", data.get("areaCode").getAsString());
dataMap.put("verifyFlag", "0".equals(data.get("farmer").getAsString()) ? "非农" : "农");
dataMap.put("countyGov", "0".equals(data.get("isGovt").getAsString()) ? "否" : "是");
// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空
JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray();
if (indusParkList != null) {
JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class);
dataMap.put("devDistrictName", jsonObject1.get("indusParkName").getAsString());
dataMap.put("devDistrict", "是");
}
}
}
httpResponse.close();
return dataMap;
} else {
System.err.println("重试次数已用完,请求失败。");
}
httpClient.close();
return null;
}
/**
* 涉农识别-新规则-标准版
*
* @param model
* @return
*/
public Object villageIdentifyStandardAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
@@ -233,10 +233,13 @@ public class ComprehensiveQueryController {
communityAvgPriceParams.put("city", model.getCity());
communityAvgPriceParams.put("address", model.getAddress());
communityAvgPriceParams.put("category", model.getCategory());
communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap()));
// communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap()));
communityAvgPriceParams.put("suggestName", String.valueOf(model.getName()));
communityAvgPriceParams.put("preciseLevel", "strict");
communityAvgPriceParams.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB");
JSONObject matchResponse = null;
try {
matchResponse = httpRequest.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8");
matchResponse = httpRequest.getJsonContent("http://open.fangjia.com/address/match", communityAvgPriceParams, "UTF-8");
// matchResponse = httpRequest.getJsonContent("http://192.168.0.19:31080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8");
} catch (HttpRequestException e) {
@@ -256,9 +259,21 @@ public class ComprehensiveQueryController {
DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(district));
if (null != statBo) {
if (statBo.getDistrictStat() != null) {
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
// 有2B价格,使用2B价格
Integer avgPrice2b = statBo.getDistrictStat().getAvg_price_2b();
if (avgPrice2b != null && avgPrice2b > 0) {
dto.setAvgPrice(avgPrice2b);
} else {
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
}
} else if (statBo.getVillaDistrictStat() != null) {
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
// 有2B价格,使用2B价格
Integer avgPrice2b = statBo.getVillaDistrictStat().getAvg_price_2b();
if (avgPrice2b != null && avgPrice2b > 0) {
dto.setAvgPrice(avgPrice2b);
} else {
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
}
}
}
@@ -638,10 +653,18 @@ public class ComprehensiveQueryController {
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
int factorPrice = evaluation.getAvgPrice();
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
Integer minPrice = (int) (evaluation.getAvgPrice() * 0.95);
Integer maxPrice = (int) (evaluation.getAvgPrice() * 1.05);
// 有2B价格用2B价格,没有用2C
Integer minPrice = 0;
Integer maxPrice = 0;
if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() > 0) {
minPrice = (int) (evaluation.getAvgPrice2b() * 0.95);
maxPrice = (int) (evaluation.getAvgPrice2b() * 1.05);
} else {
minPrice = (int) (evaluation.getAvgPrice() * 0.95);
maxPrice = (int) (evaluation.getAvgPrice() * 1.05);
}
// 偏离度超过5%不进行调整
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
if (minPrice != 0 && maxPrice != 0 && minPrice <= avgPrice && avgPrice <= maxPrice) {
evaluation.setAvgPrice(avgPrice);
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
}
@@ -657,7 +680,11 @@ public class ComprehensiveQueryController {
districtPriceMap.put("district", district.getRegion());
districtPriceMap.put("name", district.getName());
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
districtPriceMap.put("avgPrice", evaluation.getAvgPrice());
if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() != 0) {
districtPriceMap.put("avgPrice", evaluation.getAvgPrice2b());
} else {
districtPriceMap.put("avgPrice", evaluation.getAvgPrice());
}
// 根据城市等 级设置D级可信度
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
@@ -882,7 +909,11 @@ public class ComprehensiveQueryController {
otherPriceRets = new HashMap();
}
otherPriceRets.put("fjw_price", evaluation.getAvgPrice());
if (evaluation.getAvgPrice2b()!=null&&evaluation.getAvgPrice2b() > 0){
otherPriceRets.put("fjw_price", evaluation.getAvgPrice2b());
}else {
otherPriceRets.put("fjw_price", evaluation.getAvgPrice());
}
Map<String, Object> levelMap = getPriceLevel(otherPriceRets, evaluation.getCredit());
Long lastPrice = getLastPrice(levelMap, otherPriceRets, evaluation, dCredit);
@@ -55,38 +55,44 @@ public class Controller {
return mav;
}
/**
* 缓存刷新接口
* @return
*/
@Authority(role = Role.NONE)
public Object refreshCacheAction(RefreshCacheModel model){
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
Token token = FangjiaApiUtil.parse2Token(model.getToken());
if(token == null){
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
return mav;
}
final ThreadLocalData threadLocalData = ThreadLocalData.getInstance();
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
// 在验证appKey
ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId());
if(apiUser == null){
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
return mav;
}
try {
apiUserAuthorityService.findByUserId(apiUser.getId());
if(StringUtils.isNotBlank(model.getViewConfigUrl())){
apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl());
}
} catch (IllegalAccessException e) {
e.printStackTrace();
}
mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功"));
return mav;
}
/**
* 缓存刷新接口
*
* @return
*/
@Authority(role = Role.NONE)
public Object refreshCacheAction(RefreshCacheModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
Token token = FangjiaApiUtil.parse2Token(model.getToken());
if (token == null) {
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
return mav;
}
final ThreadLocalData threadLocalData = ThreadLocalData.getInstance();
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
// 在验证appKey
ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId());
if (apiUser == null) {
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
return mav;
}
try {
// 获取用户
User user = userService.findById(apiUser.getId());
if (null != user && StringUtils.isNotBlank(user.getUserName())) {
userService.findByUserName(user.getUserName());
}
apiUserAuthorityService.findByUserId(apiUser.getId());
if (StringUtils.isNotBlank(model.getViewConfigUrl())) {
apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl());
}
} catch (IllegalAccessException e) {
e.printStackTrace();
}
mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功"));
return mav;
}
@@ -6,9 +6,28 @@ import com.fangjia.open.interceptors.Role;
import com.fangjia.open.model.AddressModel;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import sun.misc.BASE64Encoder;
import java.io.IOException;
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.Map;
@@ -18,8 +37,21 @@ import java.util.Map;
@Authority(role = Role.USER)
public class GdBankController {
private static final Map<String, String> areaMap = new HashMap<String, String>() {
{
// 添加键值对到 Map 中
put("111", "主城区");
put("112", "城乡结合区");
put("121", "镇中心区");
put("122", "镇乡结合区");
put("123", "特殊区域");
put("210", "乡中心区");
put("220", "村庄");
}
};
/**
* 涉农识别-新规
* 涉农识别-新规则-光大银行
*
* @param model
* @return
@@ -31,12 +63,14 @@ public class GdBankController {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null;
String address = model.getAddress();
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
params.put("type", StringUtils.isNotBlank(model.getType()) ? model.getType() : null);
params.put("type", type);
params.put("redis", (model.getRadius() != null && model.getRadius() != 0) ? model.getRadius().toString() : null);
params.put("address", model.getAddress());
params.put("showFullValue", "2");
@@ -54,13 +88,63 @@ public class GdBankController {
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
//我们的结果
JSONObject result = json.getJSONObject("result");
if (("是").equals(result.get("countyLevelCity"))){
if (("是").equals(result.get("countyLevelCity"))) {
result.put("prefectureLevelCity", "否");
} else if(("否").equals(result.get("countyLevelCity"))){
} else if (("否").equals(result.get("countyLevelCity"))) {
result.put("prefectureLevelCity", "是");
}
basicModel.setResult(result);
// 如果是无效信息或者没有行政区划,则用我们的
if (result.containsKey("addrVerify") && ("无效".equals(result.getString("addrVerify")) || "没有行政区划".equals(result.getString("addrVerify")))) {
basicModel.setResult(result);
//我们省级及以上有开发区,直接用我们的结果
} else if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) {
//结果比较,如果涉农结果不一致
//调用齐梁 type :1 对公 0 对私
//我们 type :1 对私 2 对共
String qltype = "2".equals(type) ? "1" : "0";
Map<String, String> map = senRequest(address, qltype);
// 涉农结果不一样
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
//有市辖建制镇, 或者城市不一样 用我们
if (result.containsKey("city") && ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city"))
&& StringUtils.isNotBlank(result.getString("city"))))) {
basicModel.setResult(result);
} else {
if (("是").equals(result.get("countyLevelCity"))) {
map.put("prefectureLevelCity", "否");
} else if (("否").equals(result.get("countyLevelCity"))) {
map.put("prefectureLevelCity", "是");
}
map.put("addrVerify", "地址有效");
if (result.containsKey("countyLevelCity")) {
map.put("countyLevelCity", result.getString("countyLevelCity"));
}
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
basicModel.setResult(result);
}
} else {
basicModel.setResult(result);
}
} catch (Exception e) {
e.printStackTrace();
basicModel.setResult(result);
}
}
}
mav.setModel(basicModel);
} else {
@@ -68,4 +152,107 @@ public class GdBankController {
}
return mav;
}
private static Map<String, String> senRequest(String address, String type) throws IOException {
String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
String proxyHost = "tunnel2.qg.net";
int proxyPort = 14186;
final String proxyUsername = "113BBC06";
final String proxyPassword = "3B2FBD05B0AF";
String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}";
String authKey = proxyUsername;
String password = proxyPassword;
String auth = authKey + ":" + password;
String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8")));
CloseableHttpClient httpClient = HttpClients.createDefault();
HttpPost httpPost = new HttpPost(targetURL);
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
Authenticator.setDefault(new Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray());
}
});
httpPost.setHeader("Content-Type", "application/json");
httpPost.setHeader("Accept", "*/*");
httpPost.setHeader("Host", "146.56.217.44:9010");
httpPost.setHeader("Proxy-Authorization", authHeader);
StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON);
httpPost.setEntity(stringEntity);
CloseableHttpResponse httpResponse = null;
int retryCount = 2; // 设置重试次数
int retryDelay = 10; // 设置重试延迟时间(毫秒)
boolean success = false;
for (int i = 0; i < retryCount; i++) {
try {
httpResponse = httpClient.execute(httpPost);
success = true;
break;
} catch (IOException e) {
System.err.println("代理请求失败,进行重试...");
try {
Thread.sleep(retryDelay);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
}
}
}
if (success) {
HttpEntity responseEntity = httpResponse.getEntity();
String responseString = EntityUtils.toString(responseEntity);
// 使用 Gson 解析 JSON 字符串
Gson gson = new Gson();
JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class);
// 创建 Map 存储各个值
Map<String, String> dataMap = new HashMap<String, String>();
// 获取各个值并放入 Map
// dataMap.put("code", jsonObject.get("code").getAsString());
// dataMap.put("msg", jsonObject.get("msg").getAsString());
JsonElement dataElement = jsonObject.get("data");
if (dataElement != null && !dataElement.isJsonNull() && dataElement.isJsonObject()) {
JsonObject data = dataElement.getAsJsonObject();
// 在这里继续处理 data 对象
if (data != null && !data.isJsonNull()) {
// dataMap.put("version", data.get("version").getAsString());
dataMap.put("province", data.get("province").getAsString());
dataMap.put("city", data.get("city").getAsString());
dataMap.put("district", data.get("district").getAsString());
dataMap.put("town", data.get("town").getAsString());
dataMap.put("committee", data.get("village").getAsString());
dataMap.put("provinceCode", data.get("provinceCode").getAsString());
dataMap.put("cityCode", data.get("cityCode").getAsString());
dataMap.put("districtCode", data.get("districtCode").getAsString());
dataMap.put("townCode", data.get("townCode").getAsString());
dataMap.put("committeeCode", data.get("villageCode").getAsString());
dataMap.put("typeCode", data.get("areaCode").getAsString());
dataMap.put("verifyFlag", "0".equals(data.get("farmer").getAsString()) ? "非农" : "农");
dataMap.put("countyGov", "0".equals(data.get("isGovt").getAsString()) ? "否" : "是");
// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空
JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray();
if (indusParkList != null) {
JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class);
dataMap.put("devDistrictName", jsonObject1.get("indusParkName").getAsString());
dataMap.put("devDistrict", "是");
}
}
}
httpResponse.close();
return dataMap;
} else {
System.err.println("重试次数已用完,请求失败。");
}
httpClient.close();
return null;
}
}
@@ -10,17 +10,7 @@ import java.net.URLEncoder;
import java.nio.charset.Charset;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -30,6 +20,7 @@ import javax.validation.Valid;
import com.fangjia.core.util.*;
import com.fangjia.dal.*;
import com.fangjia.dal.mongo.dao.AliasFangDao;
import com.fangjia.dal.mongo.model.*;
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
import com.fangjia.dal.mongo.open.model.CommunityPoi;
@@ -39,6 +30,7 @@ import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.open.dto.*;
import com.fangjia.open.dto.addressGovernance.StructureDto;
import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.service.address.AddressParserUtil;
import com.fangjia.service.district.*;
import com.fangjia.service.mongo.fangjia.*;
import com.fangjia.service.open.*;
@@ -138,7 +130,6 @@ import com.mongodb.DBObject;
@Authority(role = Role.USER)
public class PropertyController {
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
private DistrictService districtService;
private AddressParser addressParser;
@@ -186,7 +177,7 @@ public class PropertyController {
private FileDao fileDao = FileDao.getInstance();
private static final int DEFAULT_PAGE_SIZE = 20;
private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias" };
private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias", "alias" };
static final int BAI_DU_GEO = 1;// 百度系坐标
static final int GAO_DE_GEO = 2;// 高德系坐标
@@ -196,6 +187,7 @@ public class PropertyController {
private DistrictRoomSqmetreRentService districtRoomSqmetreRentService;
private RentDealPriceService rentDealPriceService;
private EvaluateRatioService evaluateRatioService;
private AliasFangService aliasFangService;
private static final int EVALUATE_SHOW_DETAIL = 1;
private static final Map<String, String> PROPERTY_TYPES = new HashMap<String, String>() {
@@ -609,6 +601,25 @@ public class PropertyController {
dto.setBlock(district.getBlock());
}
dto.setPropertyType(district.getEstateType2());
if (CollectionUtils.isNotEmpty(district.getAlias())) {
// 将最相近的别名输出给用户
String alias = null;
double similarity = 0;
if (StringUtils.isNotBlank(model.getKeyword())) {
for (String mappingAlias : district.getAlias()) {
double curr = AddressParserUtil.getSimilarityRatio(mappingAlias, model.getKeyword());
if (curr > similarity ) {
similarity = curr;
alias = mappingAlias;
}
}
}
if (StringUtils.isNotBlank(alias)) {
dto.setAlias(alias);
} else {
dto.setAlias(district.getAlias().get(0));
}
}
list.add(dto);
}
@@ -2516,7 +2527,7 @@ public class PropertyController {
model.setMax(100);
}
List<String> districtNames = new ArrayList<String>();
Set<String> districtNames = new HashSet<String>();
if (null == district) {
districtNames.add(model.getName());
} else {
@@ -2524,14 +2535,56 @@ public class PropertyController {
if (district.getAlias() != null) {
districtNames.addAll(district.getAlias());
}
}
List<DealPrice> dealPrices = null;
for (String name : districtNames) {
dealPrices = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model);
if (!dealPrices.isEmpty()) {
break;
final String masterDistrictName = district.getMasterDistrictName();
if (StringUtils.isNotBlank(masterDistrictName)) {
List<District> districts;
if ("master".equals(masterDistrictName)) {
// 所有的分期数据
districts = districtService.findSlaves(model.getCity(), model.getDistrict(), model.getName());
} else {
// 根据主小区名查询所有名称
districts = districtService.findSlaves(model.getCity(), model.getDistrict(), masterDistrictName);
}
if (CollectionUtils.isNotEmpty(districts)) {
for (District districtTemp : districts) {
districtNames.add(districtTemp.getName());
}
}
// districtNames 不包含 主小区名,加进去
if (!districtNames.contains(masterDistrictName)){
districtNames.add(masterDistrictName);
}
}
}
// 将搜房中的别名与名称加入查询集合
try {
List<AliasFang> aliasFangs = aliasFangService.findFangByNameAndAlias(model.getCity(), model.getDistrict(), districtNames);
Set<String> aliasSet = new HashSet<String>();
for (AliasFang aliasFang : aliasFangs) {
if (StringUtils.isNotBlank(aliasFang.getName())) {
aliasSet.add(aliasFang.getName());
}
if (StringUtils.isNotBlank(aliasFang.getAlias())) {
aliasSet.add(aliasFang.getAlias());
}
}
districtNames.addAll(aliasSet);
} catch (Exception ex) {
ex.printStackTrace();
}
// List<DealPrice> dealPrices = new ArrayList<DealPrice>();
// 将主名与映射别名查得成交加入 存在不能查出最新成交问题
// for (String name : districtNames) {
// List<DealPrice> dealPricesTempList = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model);
// if (CollectionUtils.isNotEmpty(dealPricesTempList)) {
// dealPrices.addAll(dealPricesTempList);
// }
// }
List<DealPrice> dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), new ArrayList<String>(districtNames), model.getMax() * 5, model);
if (model.getSize() != null) {
Collections.sort(dealPrices, new Comparator<DealPrice>() {
@Override
@@ -2580,15 +2633,9 @@ public class PropertyController {
}
deal = new LinkedHashMap<String, Object>();
dealPrice.setId(null);
if (district != null) {
deal.put("city", district.getCity());
deal.put("district", district.getRegion());
deal.put("name", district.getName());
} else {
deal.put("city", dealPrice.getCity());
deal.put("district", dealPrice.getRegion());
deal.put("name", dealPrice.getDistrictName());
}
deal.put("city", dealPrice.getCity());
deal.put("district", dealPrice.getRegion());
deal.put("name", dealPrice.getDistrictName());
if (dealPrice.getArea() == null) {
deal.put("size", "");
} else {
@@ -3501,6 +3548,33 @@ public class PropertyController {
return null;
}
private List<DealPrice> findDealPricesByNames(String city, String region, List<String> names, int max,
DealPriceModel model) {
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("city", city));
params.add(new QueryParam("region", region));
params.add(new QueryParam("districtName", QueryParam.IN_OPERATOR, names));
if (model.getRoom() != null) {
params.add(new QueryParam("room", model.getRoom()));
}
if (model.getHall() != null) {
params.add(new QueryParam("hall", model.getHall()));
}
// 18年之后抓住的数据没有score字段
// params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l));
List<DealPrice> dealPrices = dealPriceService.findDealPrices(params, max);
List<DealPrice> distinctDealPrices = new ArrayList<DealPrice>();
for (DealPrice dealPrice : dealPrices) {
if (dealPrice.getAvgPrice() == null) {
dealPrice.setAvgPrice((int) (dealPrice.getTotalPrice() / dealPrice.getArea()));
}
if (!checkExists(distinctDealPrices, dealPrice)) {
distinctDealPrices.add(dealPrice);
}
}
return distinctDealPrices;
}
private List<DealPrice> findDealPricesByName(String city, String region, String name, int max,
DealPriceModel model) {
List<QueryParam> params = new ArrayList<QueryParam>();
@@ -5959,4 +6033,8 @@ public class PropertyController {
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
this.communityBasePriceService = communityBasePriceService;
}
public void setAliasFangService(AliasFangService aliasFangService) {
this.aliasFangService = aliasFangService;
}
}
@@ -0,0 +1,190 @@
//package com.fangjia.open.controllers;
//
//import com.google.gson.Gson;
//import com.google.gson.JsonArray;
//import com.google.gson.JsonObject;
//import org.apache.http.HttpEntity;
//import org.apache.http.client.methods.CloseableHttpResponse;
//import org.apache.http.client.methods.HttpPost;
//import org.apache.http.entity.ContentType;
//import org.apache.http.entity.StringEntity;
//import org.apache.http.impl.client.CloseableHttpClient;
//import org.apache.http.impl.client.HttpClients;
//import org.apache.http.util.EntityUtils;
//import sun.misc.BASE64Encoder;
//
//import java.io.IOException;
//import java.net.Authenticator;
//import java.net.InetSocketAddress;
//import java.net.PasswordAuthentication;
//import java.net.Proxy;
//import java.nio.charset.Charset;
//import java.nio.charset.StandardCharsets;
//import java.util.Base64;
//import java.util.HashMap;
//import java.util.Map;
//
////public class Test {
//// public static void main(String[] args) throws IOException {
//// // 目标URL
//// String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
////
//// // 设置代理
//// String proxyHost = "tunnel2.qg.net";
//// int proxyPort = 14186;
////
//// // 设置代理用户名和密码
//// final String proxyUsername = "113BBC06";
//// final String proxyPassword = "3B2FBD05B0AF";
////
//// // 设置请求的JSON数据
//// String address = "江苏省扬州市宝应县安宜镇宝应大道88号";
//// String type = "1";
//// String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}";
////
//// // 设置HTTP基本认证
//// String authKey = proxyUsername;
//// String password = proxyPassword;
//// String auth = authKey + ":" + password;
//// String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8")));
////
//// // 创建HTTP客户端
//// CloseableHttpClient httpClient = HttpClients.createDefault();
//// HttpPost httpPost = new HttpPost(targetURL);
////
//// // 设置代理
//// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
////
//// // 设置代理认证
//// Authenticator.setDefault(new Authenticator() {
//// protected PasswordAuthentication getPasswordAuthentication() {
//// return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray());
//// }
//// });
////
//// // 设置请求头
//// httpPost.setHeader("Content-Type", "application/json");
//// httpPost.setHeader("Accept", "*/*");
//// httpPost.setHeader("Host", "146.56.217.44:9010");
//// httpPost.setHeader("Proxy-Authorization", authHeader);
////
//// // 设置请求体
//// StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON);
//// httpPost.setEntity(stringEntity);
////
//// // 发送POST请求
//// CloseableHttpResponse httpResponse = httpClient.execute(httpPost);
////
//// // 获取响应
//// HttpEntity responseEntity = httpResponse.getEntity();
//// String responseString = EntityUtils.toString(responseEntity);
//// System.out.println(responseString);
////
//// // 关闭连接
//// httpResponse.close();
//// httpClient.close();
//// }
////}
//
//public class Test {
// public static void main(String[] args) throws IOException {
// Map<String, String> map = senRequest("江苏省泰州市靖江市通江路37号", "1");
// System.out.println(map);
//
// }
//
// private static Map<String, String> senRequest(String address, String type) throws IOException {
// String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
// String proxyHost = "tunnel2.qg.net";
// int proxyPort = 14186;
// final String proxyUsername = "113BBC06";
// final String proxyPassword = "3B2FBD05B0AF";
// String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}";
// String authKey = proxyUsername;
// String password = proxyPassword;
// String auth = authKey + ":" + password;
// String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8")));
//
// CloseableHttpClient httpClient = HttpClients.createDefault();
// HttpPost httpPost = new HttpPost(targetURL);
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
// Authenticator.setDefault(new Authenticator() {
// protected PasswordAuthentication getPasswordAuthentication() {
// return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray());
// }
// });
// httpPost.setHeader("Content-Type", "application/json");
// httpPost.setHeader("Accept", "*/*");
// httpPost.setHeader("Host", "146.56.217.44:9010");
// httpPost.setHeader("Proxy-Authorization", authHeader);
// StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON);
// httpPost.setEntity(stringEntity);
//
// CloseableHttpResponse httpResponse = null;
// int retryCount = 2; // 设置重试次数
// int retryDelay = 10; // 设置重试延迟时间(毫秒)
// boolean success = false;
// for (int i = 0; i < retryCount; i++) {
// try {
// httpResponse = httpClient.execute(httpPost);
// success = true;
// break;
// } catch (IOException e) {
// System.err.println("代理请求失败,进行重试...");
// try {
// Thread.sleep(retryDelay);
// } catch (InterruptedException ex) {
// Thread.currentThread().interrupt();
// }
// }
// }
//
// if (success) {
// HttpEntity responseEntity = httpResponse.getEntity();
// String responseString = EntityUtils.toString(responseEntity);
//
// // 使用 Gson 解析 JSON 字符串
// Gson gson = new Gson();
// JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class);
//
//
// // 创建 Map 存储各个值
// Map<String, String> dataMap = new HashMap<String, String>();
//
// // 获取各个值并放入 Map
// dataMap.put("code", jsonObject.get("code").getAsString());
// dataMap.put("msg", jsonObject.get("msg").getAsString());
// JsonObject data = jsonObject.getAsJsonObject("data");
// dataMap.put("version", data.get("version").getAsString());
// dataMap.put("province", data.get("province").getAsString());
// dataMap.put("city", data.get("city").getAsString());
// dataMap.put("district", data.get("district").getAsString());
// dataMap.put("town", data.get("town").getAsString());
// dataMap.put("village", data.get("village").getAsString());
// dataMap.put("provinceCode", data.get("provinceCode").getAsString());
// dataMap.put("cityCode", data.get("cityCode").getAsString());
// dataMap.put("districtCode", data.get("districtCode").getAsString());
// dataMap.put("townCode", data.get("townCode").getAsString());
// dataMap.put("villageCode", data.get("villageCode").getAsString());
// dataMap.put("areaCode", data.get("areaCode").getAsString());
// dataMap.put("farmer", data.get("farmer").getAsString());
// dataMap.put("isGovt", data.get("isGovt").getAsString());
// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空
// JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray();
// if (indusParkList != null) {
// JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class);
// dataMap.put("devName",jsonObject1.get("indusParkName").getAsString());
// dataMap.put("devLevel",jsonObject1.get("indusParkLevel").getAsString());
// }
//
// httpResponse.close();
// return dataMap;
// } else {
// System.err.println("重试次数已用完,请求失败。");
// }
//
// httpClient.close();
// return null;
// }
//}
//
@@ -15,6 +15,10 @@ public class PropertyDto {
private String propertyType;
private String block;
private String distance;
/**
* 新增小区别名
*/
private String alias;
public String getId() {
return id;
}
@@ -88,4 +92,12 @@ public class PropertyDto {
public void setDistance(String distance) {
this.distance = distance;
}
public String getAlias() {
return alias;
}
public void setAlias(String alias) {
this.alias = alias;
}
}
@@ -5,7 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>城市地图</title>
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
</head>
<body>
<div id="allmap" style="width: 100%;height: 100%;"></div>
@@ -6,7 +6,7 @@
<title>地图</title>
<link href="../style/bootstrap/css/bootstrap.min.css" rel="stylesheet" media="screen">
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
</head>
<body>
<div>
@@ -5,7 +5,7 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>学校地图</title>
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
</head>
<body>
<div id="allmap" style="width: 100%;height: 100%;"></div>
@@ -9,7 +9,7 @@ import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import java.util.regex.Pattern
// 分期分区与主小区映射关系表
@BatchOwner(Watcher.WangWenYan)
class FindStageDistrictBatch extends BaseBatch {
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -75,18 +75,22 @@ for(def d : datas){
map.area = d['F'].toString().toDouble()
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
map.avg_price = (int)(d['H'].toString().toDouble())
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
def trade_date = d['J']
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
if(d['I']){
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
}
// def trade_date = d['J']
// if(trade_date){
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
// }else{
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
// }
map.source = "深房家族"
map.create_date = new Date()
map.m_date = new Date()
println map
DEAL_PRICE_COLL.save(map)
}
@@ -23,7 +23,7 @@ class AddThirdPartyPrice {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def c : citys){
println c +" "+cityCodeDatas.get(c)
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c)
def rets = createDate(dists, distCodeMap)
println dists.size() +" "+rets.size()
@@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null);
def distMap = [:]
for(def d : districts){
def alias = d.alias;
@@ -58,7 +58,7 @@ class BigdataCommunityBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
def ccbMaps = null
@@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
@@ -61,13 +62,21 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
def objectId = dateToObjectId(date)
println objectId
//664a22000000000000000000 5-20
//664b73800000000000000000 5-21
def citys = CcbCommonUtils.getRunningCitys(args)
//
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -529,7 +538,7 @@ class BigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null)
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
@@ -679,6 +688,16 @@ class BigdataDealBatch extends BaseBatch{
return map
}
def updateCrawlTradeDate(){
def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20"
def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd")
def objectId = dateToObjectId(date)
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2)
def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1)
println(objectId)
CcbCommonUtils.updateTrades(startDate, endDate, objectId)
}
def addDealPriceMatch(city){
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
@@ -816,4 +835,15 @@ class BigdataDealBatch extends BaseBatch{
println "end: "+new Date()
}
def dateToObjectId(Date date) {
// 获取时间戳(单位:秒)
def timestamp = (date.time / 1000) as int
def hexTimestamp = Integer.toHexString(timestamp)
// 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值
def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000'
return new ObjectId(objectIdStr)
}
}
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
//测试通过匹配接口配房源效果
@@ -15,9 +15,10 @@ import groovy.sql.Sql
class CcbCommonUtils {
//
// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_fjw_all = ["深圳","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
@@ -69,6 +70,10 @@ class CcbCommonUtils {
citys = CcbCommonUtils.citys_252
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}else if(countStr=="all"){
citys = CcbCommonUtils.citys_fjw_all
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
}
}else{
citys = CcbCommonUtils.citys_all
@@ -428,14 +433,14 @@ class CcbCommonUtils {
return rows
}
public static def findAllDistrictByCity(def city, def cat, def fields){
public static def findAllDistrictByCity(def city, def cat, def fields, def kind){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","汤堡小区")
// q.put("name","鸿运大院")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -445,14 +450,13 @@ class CcbCommonUtils {
keys.put(field, 1);
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
def ids = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(kind){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
@@ -460,6 +464,16 @@ class CcbCommonUtils {
ids << d._id.toString()
rets << d
}
}else{
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
if(!ids.contains(d._id.toString())){
ids << d._id.toString()
rets << d
}
}
}
}
println "[${city}] 小区量:"+rets.size()
@@ -665,10 +679,6 @@ class CcbCommonUtils {
}
public static def deleteGuidedPrice(def dealdate){
def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')"
sqlInstance.executeUpdate(sql)
def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'"
pgsqlInstance.executeUpdate(sql2)
}
@@ -702,4 +712,16 @@ class CcbCommonUtils {
return map
}
public static def updateTrades(startDate, endDate, cDate){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
q.put("_id", new BasicDBObject('$gte', cDate))
def count = DEAL_PRICE_COLL.count(q)
if(count>0){
def udata = new BasicDBObject()
udata.put("trade_date", endDate)
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
}
}
}
@@ -388,19 +388,21 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
def citys = ["469005"]//CcbCommonUtils.citys_all
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
def citys = ["上海","广州"]//CcbCommonUtils.citys_all
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def cityid : citys) {
for(def city : citys) {
def cityid = cityCodeDatas.get(city)
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
println cityid +" "+rets.size()
println city +" "+rets.size()
if(rets){
set.add(cityid)
set.add(city)
}
for(def m : rets){
// def cityId = cityCodeDatas.get(city)
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
// sqlInstance236.executeUpdate(sql)
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
@@ -409,7 +411,7 @@ class DealPriceDataProcess {
total = r.count
})
if(total>0){
println cityid +" "+m.commid+" (按价格)删除:"+total
println city +" "+m.commid+" (按价格)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
pgsqlInstance.executeUpdate(sql)
}else{
@@ -419,7 +421,7 @@ class DealPriceDataProcess {
total = r.count
})
if(total>0){
println cityId +" "+m.commid+" (按小区)删除:"+total
println city +" "+m.commid+" (按小区)删除:"+total
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
pgsqlInstance.executeUpdate(sql)
}
@@ -450,7 +452,7 @@ class DealPriceDataProcess {
}
def findPeGteThirdParty2(cityid, yearmonth){
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20"
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25"
def rets = []
pgsqlInstance.eachRow(sql,{r->
def map = [:]
@@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -207,7 +207,7 @@ class GuiDingPriceBatch extends BaseBatch{
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -2,6 +2,9 @@ package com.fangjia.script.ccbevaluation
import groovy.sql.Sql
/**
* 建行使用
*/
class PgsqlCommUtils {
public static def database = "warehouse"
@@ -19,6 +22,9 @@ class PgsqlCommUtils {
}
public static def findAllDistrictCount(def cityid){
def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def count = 0
@@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null)
def fenQiMap = getFenQiMap(dists);
def rets = createCityData(dists, fenQiMap)
rows.addAll(rets)
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
@@ -1288,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
if(d['J']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
@@ -1300,22 +1301,81 @@ ExcelUtils.multiThreadingModify(fileName, {d->
// d['H'] = dist._id.toString()
// }
// }
if(!d['I']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(!d['A'] && d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
d['F'] = dist.region
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
// d['H'] = dist.name
d['G'] = dist.status
// if(d['A']=="天门" || d['A']=="迪庆"){
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
// d['K'] = val.toInteger()
// }
}
}
}
},0)
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
//ExcelUtils.multiThreadingModify(filePath, { d ->
// if(d['B']){
//
// def obj = CommonDatautils.findDistrictById(d['B'])
// if(obj){
// def markMap = obj.mark_map
// if(!obj.mark_map){
// markMap = [:]
// }
// def data = new BasicDBObject()
// markMap.put("exclusive", "是")
// data.put("mark_map", markMap)
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
// }
// }
//
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// if(d['B'] && !d['D']){
// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
// if(jsonObject != null && jsonObject.credit>0) {
// d['D'] = jsonObject.get("city")
// d['E'] = jsonObject.get("district")
// d['F'] = jsonObject.get("name")
// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
// if(dbo){
// d['G'] = dbo.status
// d['H'] = dbo.name_cat
// d['I'] = dbo.estate_type2
// d['J'] = "接口匹配"
// }
// }
//
//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
//// if(!objs){
//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
//// }
//// if(objs && objs.size()==1){
//// def o = objs[0]
//// if(o){
//// d['D'] = o.city
//// d['E'] = o.region
//// d['F'] = o.name
//// d['G'] = o.status
//// d['H'] = o.name_cat
//// d['I'] = o.estate_type2
//// d['J'] = "别名"
//// }
//// }
//
//
// }
//})
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
@@ -55,7 +55,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
// }
// }
//
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
@@ -71,7 +71,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
def statMap = getDistrictStatByCity(city, weekend)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
def thirdPriceMap = findThirdPartyPrice(city, statMap)
//def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用)
@@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{
def size = regionDatas.size()
final CountDownLatch latch = new CountDownLatch(size)
def totalCount = new AtomicLong()
for(def i = 0;i<regionDatas.size();i++){
final def arr = regionDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try {
totalCount.addAndGet(1)
println totalCount.get() +" /${size} "+arr[0] + " "+ arr[1]
def row = createDatas(arr[0], arr[1], startDate, endDate, cityCodeDataMap, arr[2])
if(row){
rets << row
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
// } catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// };
// })
}
latch.await()
// latch.await()
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTPRICE.dat"
@@ -206,12 +207,14 @@ class BigdataCitypriceBatch extends BaseBatch{
def allDistRegionPreMap = [:]
def allLastMonthMap = [:]
def allLastYearMap = [:]
def allCcbPriceMap = [:]
def allCcbPriceCaseCountMap = [:]
for(def city : citys) {
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
allDistricts.addAll(districts)
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
allPriceMap = allPriceMap + priceMap
// def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
// allPriceMap = allPriceMap + priceMap
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
allRentMap = allRentMap + rentMap
@@ -220,9 +223,15 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
allDealCountMap = allDealCountMap + dealCountMap
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
def datasMap = findCommunityPriceByCityMap(cityCode, startDate.toString().substring(0,6))
allCcbPriceMap = allCcbPriceMap + datasMap
def caseCountMap = findCommunityPriceCountByCityMap(cityCode, startDate.toString().substring(0,6))
allCcbPriceCaseCountMap = allCcbPriceCaseCountMap + caseCountMap
def cityRankMap = findCommunityCityRank(datas)
allCityRankMap = allCityRankMap + cityRankMap
@@ -245,7 +254,7 @@ class BigdataCitypriceBatch extends BaseBatch{
allLastYearMap = allLastYearMap + lastYearMonthMap
}
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
if(cityLists){
rets.addAll(cityLists)
}
@@ -263,7 +272,7 @@ class BigdataCitypriceBatch extends BaseBatch{
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
def totalCount = new AtomicLong()
def distCount = districts.size()
final CountDownLatch latch = new CountDownLatch(distCount)
@@ -276,7 +285,7 @@ class BigdataCitypriceBatch extends BaseBatch{
void run() {
try {
totalCount.addAndGet(1)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
if(row){
rets << row
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
@@ -323,37 +332,59 @@ class BigdataCitypriceBatch extends BaseBatch{
}
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
def sellDataType = 1
def rentDataType = 1
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
//房源查询是否按建行均价筛选房源
def sellMap = null//findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
def sell_avg_price = null
def sell_avg_area = null
def sell_avg_total = null
def sellStartDate = null
def sell_count = 0
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
sellDataType = 6
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
if(allCcbPriceMap.containsKey(dist._id.toString())){
sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
// sell_avg_area = avgValue(sellMap.area)
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
if(allCcbPriceCaseCountMap.containsKey(dist._id.toString())){
sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
}
}
if(sell_count){
sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
sellDataType = 6
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
}
}
}
if(sellMap){
sell_count = sellMap.avg_price.size()
if(sell_count>3){
sell_avg_price = avgValue(sellMap.avg_price)
if(sellMap){
sell_avg_area = avgValue(sellMap.area)
sell_avg_total = avgValue(sellMap.total_price)
}else{
sell_count = 0
sellDataType = null
sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
// if(sell_count>3){
// if(allCcbPriceMap.containsKey(dist._id.toString())){
// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
// sell_avg_area = avgValue(sellMap.area)
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
// }else{
// sell_count = 0
// sellDataType = null
// }
// }else{
// sell_count = 0
// sellDataType = null
// }
}
}
def rent_avg_price = null
def rent_avg_area = null
def rent_avg_total = null
@@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{
rentDataType = null
}
}
if(!sell_avg_price){
sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
}
//三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格
// if(!sell_avg_price){
// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
// }
if(!rent_avg_price){
rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
}
@@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{
def sell_avg_area = null
def sell_avg_total = null
def sellStartDate = null
def avgPrice = null
if(region){
avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6))
}else{
avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6))
}
if (sell_count < 10) {
sellDataType = 3
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
@@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{
sellStartDate = startDate
}
if (sell_count) {
sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area")
sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price")
}
@@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{
return datas;
}
def findCommunityPriceByCityMap(cityCode, month){
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def dataMap = [:]
pgsqlInstance.eachRow(sql, { r ->
dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc)
})
return dataMap;
}
def findCommunityPriceCountByCityMap(cityCode, month){
def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}";
def dataMap = [:]
pgsqlInstance.eachRow(sql, { r ->
if(r.gujiabdsrt3mlgcasenum){
dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum)
}
})
return dataMap;
}
def findCcbCityPrice(cityCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
def avgPrice = null
@@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{
return regionPriceMap;
}
def findCcbRegionPriceByRegion(cityCode, regionCode, month){
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'";
def price = null
pgsqlInstance.eachRow(sql, { r ->
price = r.gujia_bds_clc_avprc
})
return price;
}
def districtCityPrictRate(cityPrice, datas){
def distCityRateMap = [:]
@@ -159,7 +159,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
}
@@ -246,7 +246,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
}
@@ -304,7 +304,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
}
@@ -362,7 +362,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
}
@@ -425,7 +425,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
}
@@ -504,7 +504,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
}
@@ -590,7 +590,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
rets << ret
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
}
@@ -4,17 +4,15 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import java.sql.ResultSet
import java.sql.ResultSetMetaData
/**
* 建行融合数据导出
*/
@BatchOwner(Watcher.GaoJuan)
class ExportDistrictRongHe extends BaseBatch{
@@ -239,7 +237,7 @@ class ExportDistrictRongHe extends BaseBatch{
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'";
def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
@@ -255,23 +253,23 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "EVALUATE_RATE_CITY_FJW_ADD 数据文件输出完成......"
println "EVALUATE_RATE_CITY_FJW_HIS_ADD 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "EVALUATE_RATE_CITY_FJW_ADD 上传Ftp完成......"
println "EVALUATE_RATE_CITY_FJW_HIS_ADD 上传Ftp完成......"
}
def exportEvaluateRateComm(queryDate){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' ";
def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' ";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
@@ -284,6 +282,7 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.unitprc_avg
data << r.comm_type
data << r.area_avg_coef
data << r.unitprc_avg_coef
data << r.basic_price_coef
data << r.buildarea_coef
data << r.type_coef
@@ -296,16 +295,16 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat"
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 数据文件输出完成......"
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 上传Ftp完成......"
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 上传Ftp完成......"
}
@@ -1,6 +1,7 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
@@ -8,11 +9,18 @@ import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.AggregationOutput
import com.mongodb.client.MongoCursor
import com.mongodb.client.model.Accumulators
import com.mongodb.client.model.Aggregates
import com.rabbitmq.client.impl.AMQImpl
import groovy.json.JsonSlurper
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.poi.ss.util.RegionUtil
import org.bson.Document
import org.bson.types.ObjectId
import com.fangjia.core.util.HttpRequest;
@@ -110,6 +118,8 @@ class CommonDatautils {
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
@@ -267,7 +277,7 @@ class CommonDatautils {
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","揭흩소蹈")
// q.put("name","托斯卡纳庄园")
DBObject keys = new BasicDBObject();
if(fields){
@@ -276,7 +286,7 @@ class CommonDatautils {
}
}
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -1459,6 +1469,28 @@ class CommonDatautils {
return obj;
}
static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){
DBObject q = new BasicDBObject();
if(region){
q.put('region', region)
}
q.put('city',city)
q.put('cat', cat)
q.put('status',0)
BasicDBObject query = new BasicDBObject();
query.putAt(QueryOperators.IN, [name]);
q.put("mapping_alias", query);
def dbcursor = SEAWEED_COLL.find(q);
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
static def findSeaweeds(city, cat, name_cat){
DBObject q = new BasicDBObject();
q.put('city',city)
@@ -1691,7 +1723,34 @@ class CommonDatautils {
}
return regionMap;
}
public static def getAllCityFullRegions(){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("category","region");
//queryCondition.put("show",1);
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
def regionMap = [:];
for(DBObject obj : dbCursor){
def city = obj.get("city");
def region = obj.get("name");
def pattern = obj.get("pattern");
if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) {
def r_map = regionMap.get(city);
if(!r_map){
r_map = [:]
regionMap.put(city, r_map);
}
if(!pattern){
pattern = [];
pattern << region;
}
r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]);
}
}
return regionMap;
}
public static def getRegionsByCity(def city){
@@ -1834,6 +1893,25 @@ class CommonDatautils {
def dbo = null;
while(cur.hasNext()) {
dbo = cur.next()
if(dbo.value_type == 1){
dbo.cal_type = "指导价"
}else if(dbo.value_type == 2){
dbo.cal_type = "自身样本"
}else if(dbo.value_type == 3){
dbo.cal_type = "第三方"
}else if(dbo.value_type == 4){
dbo.cal_type = "交易数据"
}else if(dbo.value_type == 5){
dbo.cal_type = "周边楼盘"
}else if(dbo.value_type == 6){
dbo.cal_type = "推导"
}else if(dbo.value_type == 9){
dbo.cal_type = "神经网络"
}else if(dbo.value_type == 10){
dbo.cal_type = "无方式"
}else{
dbo.cal_type = ""
}
}
return dbo;
}
@@ -4496,6 +4574,28 @@ class CommonDatautils {
return townMap
}
static def findAllTownRanges(){
def q = new BasicDBObject()
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("town", 1)
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
def townMap = [:]
for(def d : dbcursor){
def lists = townMap.get(d.city)
if(!lists){
lists = []
}
def t = [:]
t.region = d.region
t.town = d.town
lists << t
townMap.put(d.city, lists)
}
return townMap
}
static def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
@@ -4530,6 +4630,108 @@ class CommonDatautils {
}
static def queryFullNameByCity(){
def q = new BasicDBObject()
q.put("type","城市")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def cityMap = [:]
for(def d : dbcursor){
def city = RegionData.formatCity(d.city)
if(city){
cityMap.put(city, d)
}else{
//println d.city +" 格式化不出"
}
}
return cityMap
}
static def queryDistNameByCity(){
def q = new BasicDBObject()
q.put("type","区域")
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
def regionMap = [:]
for(def d : dbcursor){
if(d.mzb_district_name){
def regions = regionMap.get(d.city)
if(!regions){
regions = []
}
regions << d
regionMap.put(d.city, regions)
}else{
println d.province+" "+d.district +" 格式化不出民政部区域"
}
}
return regionMap
}
static def queryCommunityByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("province",new BasicDBObject('$in', ['福建省','重庆市']))
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
def townMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
return townMap
}
static def queryNewTownByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
def townMap = [:]
for(def d : dbcursor){
def key = d.city
def list = townMap.get(key)
if(!list){
list = []
}
def t = [:]
t.region = d.district
t.town = d.street
list << t
townMap.put(key, list)
}
return townMap
}
static def queryDistNameCity(){
def q = new BasicDBObject()
@@ -4600,5 +4802,55 @@ class CommonDatautils {
}
static def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
}
@@ -0,0 +1,3 @@
package com.fangjia.script.fuzhou
@@ -283,7 +283,12 @@ class DistrictSellStatBatch extends BaseBatch {
def dbcursor = DISTRICT_SELL_STAT.find(q)
def priceMap = [:]
for(def d : dbcursor){
priceMap.put(d.region +"##"+d.district_name, d)
if(d.estate_type=="别墅"){
priceMap.put(d.region +"##"+d.district_name+"##别墅", d)
}else{
priceMap.put(d.region +"##"+d.district_name+"##小区", d)
}
}
return priceMap
}
@@ -29,11 +29,11 @@ class FzDataResultStat {
static main(args) {
def task = new FzDataResultStat()
// task.priceMarkupRun();
task.priceMarkupRun();
// task.regionPriceTrendRun()
//// task.margeExcel()
// task.saveCsvData()
task.testNewPrice()
// task.testNewPrice()
}
def margeExcel(){
@@ -58,7 +58,7 @@ class FzDataResultStat {
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
// List<String> list1= FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// def dists = CommonDatautils.findAllDistricts("¸£ÖÝ", "district", ["status"])
@@ -82,7 +82,7 @@ class FzDataResultStat {
}
// List<String> list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(lastMonth)
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
def map2 = [:]
for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name
@@ -22,8 +22,8 @@ class FzDistrictMatch {
// task.addPrice()
// task.addThirdPrice()
// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
task.longyanPrice()
task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
// task.longyanPrice()
}
def longyanPrice(){
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -238,6 +238,7 @@ class FiledStrand {
}
static def getValue(def val){
val = val.replace(",","")
val = getNumber(val);
if(val){
if(val.indexOf(".")>-1 && !val.toString().endsWith(".0")){
@@ -1,23 +1,115 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
class DevProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
def task = new DevProcess();
task.run()
// task.run()
// task.parseCityAndDist()
task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
// task.calAddrDistince()
// task.updateCityAndDist()
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
// task.addDevAddres2()//添加相差2公里范围内的地址
// task.matchDevAndAddAddress()
// task.stat()
// task.addNewDev()
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo43()
// task.addNewDev2()
// task.addAlias()
// task.addDevShape()//添加开发区围栏
// task.matchDevTownship()//添加乡镇街道
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
}
def parseCityAndDist(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
if(!d['J']){
// def obj = getMapInfo(null, d['A']+d['C'])
// if(obj){
// d['E'] = obj.city.toString()
// d['F'] = obj.district.toString()
// d['G'] = obj.formatted_address.toString()
// d['H'] = obj.province.toString()
// }
def obj = FangjiaClientUtils.formatNinelevel(null, d['C']);
if(obj) {
def jsonObject = obj.get("result")
if (jsonObject) {
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
d['J'] = jsonObject.get("province")
}
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
d['K'] = jsonObject.get("city")
}
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
d['L'] = jsonObject.get("district")
}
}
}
}
})
}
def getMapInfo(city, address){
def json = GaoDeApiUtils.geo(city, address)
println json
if(json && json.status=="1"){
def ret = json.geocodes
if(ret){
def obj = ret.get(0)
def str = obj.location
def level = obj.level
// if(str && ["门牌号","单元号","道路","道路交叉路口","热点商圈","村庄","兴趣点","门址","住宅区"].contains(level)){
def arrs = str.split(",")
return ["lng":arrs[0], "lat":arrs[1],"province":obj.province,"city":obj.city, "district":obj.district, "formatted_address":obj.formatted_address]
// }else{
// println address +" "+level
// return null
// }
}
}
}
def run(){
def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls")
@@ -77,10 +169,611 @@ class DevProcess {
}else{
println udata.name +" 已存在"
}
}
}
}
def findDev(city, name){
def q = new BasicDBObject()
q.put("name", name)
q.put("city", city)
def obj = development_zone_CONN.findOne(q)
return obj
}
def devSplitRoad(){
def i = 0;
while(i<20){
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['E'] && !d['G']){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
if(jsonObject1) {
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
def devName = jsonObject1.get("dev")
def obj = findDev(d['B'], devName)
d['F'] = devName
if (obj) {
d['Q'] = obj._id.toString()
d['R'] = obj.name
}
}
}
}
if(d['C'] && !d['S']) {
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
d['G'] = jsonObject.get("province")
}
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
d['H'] = jsonObject.get("city")
}
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
d['I'] = jsonObject.get("district")
}
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
d['J'] = jsonObject.get("township")
}
if (jsonObject && street && streetNumber && street != "null") {
d['K'] = jsonObject.get("street")
def road = jsonObject.get("street")
if (streetNumber != 'null') {
d['L'] = jsonObject.get("streetNumber")
road = road + jsonObject.get("streetNumber")
} else {
d['N'] = "信息不完整"
}
if (!d['N'] && !d['C'].contains(road)) {
d['N'] = "原地址不含路"
}
if (!d['N']) {
def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$')
if (!flag) {
d['N'] = "路非法"
}
def roadsize = jsonObject.get("street").size()
if (!d['N'] && (roadsize < 3 || roadsize > 7)) {
d['N'] = "路太长|路太短"
}
}
}else{
d['N'] = "无地址"
}
if (jsonObject.get("dev") && jsonObject.get("dev") != "null") {
def devName = jsonObject.get("dev")
def obj = findDev(d['B'], devName)
d['M'] = devName
if (obj) {
d['S'] = obj._id.toString()
d['T'] = obj.name
}
}
}
},0)
i++
}
}
}
def devNameSplitRoad(){
def fileName = "/home/goojia/gj/company.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\开发区\\56万需修改地址.xlsx"
}
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['P'] && d['P']=="路正常" && !d['O']){
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
if(jsonObject){
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['J'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['K'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['L'] = jsonObject.get("district")
}
if(jsonObject && street && streetNumber && street!="null"){
d['M'] = jsonObject.get("street")
if(streetNumber!='null'){
d['N'] = jsonObject.get("streetNumber")
}
}
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
d['O'] = jsonObject.get("dev")
}
}
}
},0)
}
def addFlag(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
if(d['H'] && d['D']){
if(d['D'].contains(d['H'])){
d['O'] = "是"
}else{
def newName = null
if(d['H'].toString().startsWith(d['I'])){
newName = d['H'].toString().substring(d['I'].toString().length())
}else if(d['H'].toString().startsWith(d['I']+"省")){
newName = d['H'].toString().substring(d['I'].toString().length()+1)
}
if(newName && d['D'].contains(newName)){
d['O'] = "是2"
}
}
}
},1);
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
if(!d['Q'] && d['L']){
def q = new BasicDBObject()
q.put("name", d['L'])
q.put("city",d['G'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['Q'] = obj._id.toString()
d['R'] = obj.name
}
}
},0);
}
def calAddrDistince(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
if(d['M'] && d['N'] && d['Y'] && d['Z']){
def addr1 = getFullAddress(d['J'], d['K'],d['L'],d['M'] , d['N'])
def addr2 = getFullAddress(d['V'], d['W'], d['X'], d['Y'] , d['Z'])
def json1 = GaoDeApiUtils.geo(d['K'], addr1)
def json2 = GaoDeApiUtils.geo(d['W'], addr2)
def arrs1 = getLocation(json1)
def arrs2 = getLocation(json2)
if(arrs1 && arrs2){
def distance = GeoUtils.calculateDistance(arrs1[0].toString().toDouble(), arrs1[1].toString().toDouble(), arrs2[0].toString().toDouble() , arrs2[1].toString().toDouble() )
d['AA'] = distance
}
}
},0);
}
def getFullAddress(province, city, region, street, streetName){
def addr = ""
if(addr){
addr = province
}
if(city){
addr = addr + city
}
if(region){
addr = addr + region
}
addr = addr + street + streetName
return addr
}
def getLocation(json1){
def arrs1 = null
if(json1 && json1.status=="1") {
def ret = json1.geocodes
if (ret) {
def str = ret.get(0).location
if(str){
arrs1 = str.split(",")
}
}
}
return arrs1
}
def updateCityAndDist(){
ExcelUtils.modify("D:\\数据\\数据建设\\等级开发区 - 汇总.xlsx",{d->
if(d['L']){
def udata = new BasicDBObject()
if(d['L']=="采用拆分结果"){
if(d["I"]){
udata.put("province", d["I"])
}
if(d["J"]){
udata.put("city", d["J"])
}
if(d["K"]){
udata.put("district", d["K"])
}
}else if(d['L']=="采用地图结果"){
if(d["H"]){
udata.put("province", d["H"])
}
if(d["E"]){
udata.put("city", d["E"])
}
if(d["F"]){
udata.put("district", d["F"])
}
}
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['C']))
if(obj && (!obj.city && !obj.district)){
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
}
})
}
def addDevAddres1(){
def rets = []
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
if(d['O'] && d['P']=="正常" && d['U']){
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
if(obj){
d['AB'] = "存在"
def addr_alias = obj.addr_alias
if(!addr_alias){
addr_alias = []
}
def old_count = addr_alias.size()
if(!addr_alias.contains(d['M']+d['N'])){
addr_alias << d['M']+d['N']
}
if(old_count != addr_alias.size()){
def udata = new BasicDBObject()
udata.put("addr_alias", addr_alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
}else{
rets << d
println "匹配:"+d['U']+" 原始:"+d['H']
}
}
},0)
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def addDevAddres2(){
def rets = []
ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d->
if((d['R'] && d['R'].toString()=="正常") || (d['P'] && d['P'].toString()=="正常")){
def obj = null
if(d['U']){
try{
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
}catch(Exception e){
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
}
}
if(!obj){
try{
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H']))
}catch(Exception e){
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H']))
}
}
if(obj){
d['AB'] = "存在"
def addr_alias = obj.addr_alias
if(!addr_alias){
addr_alias = []
}
def old_count = addr_alias.size()
if(d['R'] && d['R']=="正常" && !addr_alias.contains(d['Y']+d['Z']) && d['X'] && d['F'] && d['X']==d['F']){
addr_alias << d['Y']+d['Z']
}
if(d['P'] && d['P']=='正常' && !addr_alias.contains(d['M']+d['N']) && d['F'] && d['L'] && d['L']==d['F']){
addr_alias << d['M']+d['N']
}
if(old_count != addr_alias.size()){
def udata = new BasicDBObject()
udata.put("addr_alias", addr_alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
}else{
// rets << d
println "匹配:"+d['U']+" 原始:"+d['H']
}
}
}, 0)
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def stat(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
def total = 0
for(def d : dbcursor){
if(d.addr_alias){
total = total + d.addr_alias.size()
println d.addr_alias.size()
}
}
println total
}
def addNewDev(){
ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d->
def udata = new BasicDBObject()
udata.address = d['M']+d['N']
udata.create_time = new Date()
if(d['G']){
if(d['G'].contains("国家级")){
udata.level = "国家级"
}else if(d['G'].contains("省级")){
udata.level = "省级"
}
}
udata.name = d['H']
udata.province = d['J']
udata.city = d['K']
udata.district = d['L']
udata.type = d['G']
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
if(!obj){
development_zone_CONN.save(udata)
}else{
println udata.name +" 已存在"
}
})
}
def addNewDev2(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
if(d['F'] && d['F']=="新增"){
def udata = new BasicDBObject()
udata.create_time = new Date()
udata.province = d['G']
udata.city = d['I']
udata.district = d['J']
udata.level = "省级"
udata.name = d['D']
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
if(!obj){
development_zone_CONN.save(udata)
}else{
println udata.name +" 已存在"
}
}
})
}
def createDevMapping(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
for(def d : dbcursor){
def formatAddress = null
if(d.address){
formatAddress = AddressUtil.formatAddress(d.city, d.address)
}
if(formatAddress && !formatAddress.endsWith("号") && !formatAddress.endsWith("弄")){
formatAddress = null
}
def addr_alias = []
if(formatAddress){
addr_alias << formatAddress
}
if(d.addr_alias){
addr_alias = addr_alias + d.addr_alias
addr_alias.unique()
}
if(addr_alias){
addr_alias.each{addr->
def udata = new BasicDBObject()
udata.put("province", d.province)
udata.put("city", d.city)
udata.put("district", d.district)
udata.put("name",d.name)
udata.put("type", "地址")
if(d.township){
udata.put("street", d.township)
}
udata.put("address", addr)
DEV_MAPPING_CONN.save(udata)
}
}
}
}
def deleteAddr(){
ExcelUtils.modify("D:\\数据\\数据建设\\删除开发区地址.xlsx",{d->
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", new ObjectId(d['A'])))
});
}
def deleteAddr2(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d->
if(d['Q'] && d['M']){
def q = new BasicDBObject()
q.put("_id", new ObjectId(d['Q']))
def obj = development_zone_CONN.findOne(q)
if(obj){
def q2 = new BasicDBObject()
q2.put("name", obj.name)
q2.put("address", d['I']+d['J'])
def sameObj = DEV_MAPPING_CONN.findOne(q2)
if(sameObj){
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id))
}
}
}
});
}
def synchDevTo43(){
def dbcursor = development_zone_CONN.find()
def rets = []
for(def r : dbcursor){
rets << r
}
DEVELOPMENT_ZONE_43_COLL.insert(rets)
}
def matchDevAndAddAddress(){
ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d->
if(d['R']){
def q = new BasicDBObject()
q.put("name", d['R'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['P'] = obj._id.toString()
d['Q'] = obj.name
def address = d['L']+d['M']
q.put("address", address)
// def street = d['G']
// q.put("township", street)
def sameObj = DEV_MAPPING_CONN.findOne(q)
if(!sameObj){
def udata = new BasicDBObject()
udata.put("province", obj.province)
if(obj.city){
udata.put("city", obj.city)
}
if(obj.district){
udata.put("district", obj.district)
}
udata.put("name",obj.name)
udata.put("type", "地址")
udata.put("address",address)
// udata.put("type", "乡镇街道")
// udata.put("township", street)
DEV_MAPPING_CONN.save(udata)
println obj.name +" "+address
}
}
}
},0);
}
def matchDevTownship(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
if(d['C']){
def q = new BasicDBObject()
q.put("name", d['C'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['E'] = obj._id.toString()
def street = d['D']
q.put("township", street)
def sameObj = DEV_MAPPING_CONN.findOne(q)
if(!sameObj){
def udata = new BasicDBObject()
udata.put("province", obj.province)
if(obj.city){
udata.put("city", obj.city)
}
if(obj.district){
udata.put("district", obj.district)
}
udata.put("name",obj.name)
// udata.put("type", "乡镇街道")
// udata.put("township", street)
udata.put("type", "区域")
udata.put("dev_district", street)
udata.put("c_date",new Date())
DEV_MAPPING_CONN.save(udata)
println obj.name +" "+street
}
}
}
},1);
}
def addAlias(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d->
def q = new BasicDBObject()
q.put("name", d['A'])
def obj = development_zone_CONN.findOne(q)
if(obj){
d['D'] = obj._id.toString()
def arrs = d['B'].split("、")
def alias = obj.alias
if(!alias){
alias = []
}
for(def s : arrs){
if(!alias.contains(s)){
alias << s
}
}
def udata = new BasicDBObject()
udata.put("alias", alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}
})
}
def addDevShape(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d->
if(d['W']){
def s = d['S']
JSONObject jsonObject = JSONObject.fromObject(s);
Map<String, Object> map = DataHandleUtils.JsonToMap(jsonObject);
def udata = new BasicDBObject()
udata.put("shape", map)
development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata))
}
})
}
def addDevStreet(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
if(d['J']=="是"){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道"))
for(def obj : dbcursor){
def o = new BasicDBObject()
o.put("province" , obj.province)
o.put("city" , obj.city)
o.put("district" , obj.mzb_district_name)
o.put("name" , d['E'])
o.put("type" , "乡镇街道")
o.put("township" , obj.street)
DEV_MAPPING_CONN.save(o)
}
}
})
}
}
@@ -0,0 +1,45 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
class DistrictAliasProcess {
static main(args) {
def task = new DistrictAliasProcess();
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d->
if(d['H']=="是"){
def udata = new BasicDBObject()
udata.put("status", 1)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
}else if(d['H']=="否"){
def aliasDist = d['F']
def arrs = aliasDist.split("-")
def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district")
if(obj){
def alias = obj.alias
def mapping_alias = obj.mapping_alias
def udata = new BasicDBObject()
if(alias && alias.contains(d['D'])){
alias.remove(d['D'])
udata.put("alias", alias)
}
if(mapping_alias && mapping_alias.contains(d['D'])){
mapping_alias.remove(d['D'])
udata.put("mapping_alias", mapping_alias)
}
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
}
}
})
}
}
@@ -6,6 +6,7 @@ import com.fangjia.script.ccbevaluation.PgsqlCommUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
@@ -32,7 +33,9 @@ class GXDdistrictMatch {
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
task.restoreDist()//恢复小区status=0
// task.restoreDist()//恢复小区status=0
task.runMatch()
}
def addOldMatch(){
@@ -133,6 +136,22 @@ class GXDdistrictMatch {
})
}
def runMatch(){
def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx
def city = "福州"
ExcelUtils.modify(fileName, { d ->
println d['C']
if(d['B'] && d['A']){
def region = RegionData.formatRegion(city, d['B'])
if (region) {
d = match(city, region, d)
} else {
println d['A'] + "#" + d['E'] + " 区域无法格式化"
}
}
})
}
def updateDistStatus(){
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
def index = 0
@@ -187,27 +206,32 @@ class GXDdistrictMatch {
}
def saveToDB(){
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
ExcelUtils.modify(fileName, { d ->
if(d['U'] && d['AG']!='过滤'){
if(d['A']){
def udata = new BasicDBObject()
udata.put("city", d['AD'])
udata.put("region", d['AE'])
def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("city", d['A'])
udata.put("region", d['B'])
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
udata.put("name", name)
def alias = []
if(name!=d['C']){
alias << d['C']
}
if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
alias << d['AI']
if(d['G'] && name!=d['G']){
alias << d['G']
}
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
// alias << d['AI']
// }
if(alias){
udata.put("alias", alias)
}
udata.put("address", d['AF'])
def addr = AddressUtil.formatAddress(d['A'], d['D'])
if(addr){
udata.put("address", addr)
}
udata.put("status", "0")
udata.put("name_cat", "0")
udata.put("cat", "district")
@@ -215,11 +239,11 @@ class GXDdistrictMatch {
def estateType2 = "普通住宅"
def type = ""
if(d['AH']){
type = d['AH']
}else{
type = d['Y']
}
// if(d['AH']){
// type = d['AH']
// }else{
// type = d['Y']
// }
if(type){
if(type.contains("别墅")){
estateType2 = "别墅"
@@ -228,12 +252,12 @@ class GXDdistrictMatch {
}
}
udata.put("estateType2", estateType2)
if(d['AA']){
Long lng = d['AA']*10000000000
if(d['F']){
Long lng = d['F']*10000000000
udata.put("lng", lng)
}
if(d['AB']){
Long lat = d['AB'] * 10000000000
if(d['E']){
Long lat = d['E'] * 10000000000
udata.put("lat", lat)
}
@@ -265,7 +289,7 @@ class GXDdistrictMatch {
if(!d['U']){
d = matchApi(city, region, d, "property", false)
if(!d['U']){
d = matchApi(city, region, d, null, true)
d = matchApi(city, region, d, null, false)
}
}
@@ -285,14 +309,18 @@ class GXDdistrictMatch {
}
def matchApi(city, region, d, cat, useMap){
def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
if(!jsonObject){
def jsonObject = null
if(!jsonObject && d['H']){
jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
}
if(!jsonObject && d['C']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
if(!jsonObject && d['D']){
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
}
}
if(jsonObject != null && jsonObject.credit>0) {
if(jsonObject != null) {
d['U'] = "接口匹配"
d['V'] = jsonObject.get("city")
d['W'] = jsonObject.get("district")
@@ -0,0 +1,317 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
static main(args) {
def task = new PoiLocation();
task.run();
}
def run(){
def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"]
for(def city : citys){
findPois(city)
}
}
def findPois(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("run_status", new BasicDBObject('$exists', false))
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
def index = 0
for(def d : dbcursor){
rets << d
if(rets.size()>=10000){
index++
def datas = checkData(index, rets)
// updateToEs(datas)
updateToEs2(datas)
updateToMG(datas)
rets = []
print "."
}
}
return rets;
}
def checkData(def index, def datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def map = running(d)
rets << map
if(map.update_status && map.update_status!=100){
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
}
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def running(d){
def regions = getFullRegion(d)
def town = null
def map =[:]
if(d.location){
//判断区域否在围栏里
def regionFlag = null
if(d.district){
for(def rs : regions){
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
if(regionFlag){
//判断township是否在统计局中
if(d.township){
town = getSimilarTown(regions, d.city, d.township)
}
if(town){
if(town.street == d.township){
//打标记
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = town.street
}
}else{
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
if (mapTown) {
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = mapTown.street
}else{
if(d.district!=gaodeMap.district){
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}else{
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
}
}
}else{
//打标记“经纬度有误,不在区域围栏中”
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
map.run_status = 1 // 1:已运行
map.id = d._id.toString()
map.township = ""
}
}
return map;
}
def getFullRegion(d){
def regions = regionMap.get(d.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.district)){
names << r
}
}
}
return names;
}
def getSimilarTown(regions, city, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(city+"#"+r.district)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def updateMapPoi(def id, def township){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.township='${township}'",
"lang": "painless"
},
"query": {
"term": {
"id": "${id}"
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
println(response)
}
def updateToEs2(rows) {
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if (map.update_status && map.update_status !=100) {
updateMapPoi(map.id, map.township)
}
}
}
def updateToEs(rows){
BulkRequest request = new BulkRequest();
int batchSize = 1000;
int processedCount = 0;
def client = ElasticUtils.getClient89("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
if(map.update_status && map.update_status==2){
map.remove("run_status")
map.remove("update_status")
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
// 批量提交数据
if (request.numberOfActions() >= batchSize) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
request = new BulkRequest();
}
}
}
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
println "结束时间 "+new Date()
}
def updateToMG(rows){
def rets = []
for(def i=0;i<rows.size();i++){
def m = rows[i]
def udata = new BasicDBObject()
m.each{k,v->
if(k!="id"){
udata.put(k,v)
}
}
def q = new BasicDBObject()
q.put("_id", m.id)
if(udata){
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
}else{
println m.toString()+" 无法修改-无值"
}
}
}
}
@@ -0,0 +1,78 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class TempExportData {
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new TempExportData()
task.run()
}
def run() {
def citys = CommonDatautils.findAllCity()
def rets = []
rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"]
for (def city : citys) {
def cityRets = getResults(city)
rets.addAll(cityRets)
}
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
}
def getResults(city) {
def rets = []
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
final CountDownLatch latch = new CountDownLatch(dists.size())
final AtomicInteger count_ = new AtomicInteger(0)
for (def i = 0; i < dists.size(); i++) {
def d = dists[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.addAndGet(1)
try {
def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district")
if (objs) {
def ret = []
ret << d._id.toString()
ret << d.city
ret << d.region
ret << d.name
ret << d.address
def str = ""
for (def o : objs) {
str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n"
}
ret << str
ret << objs.size()
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
println(city +" "+rets.size())
return rets;
}
}
@@ -0,0 +1,105 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class Test{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
def task = new Test()
task.run()
}
def run(){
def i=1
while(i<2){
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
def rets = process(districts)
println i + " "+districts.size() +" "+rets.size()
i++
}
}
def process(districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = new CopyOnWriteArrayList()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
methodA(rMap)
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
rets << rMap.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return rets
}
def methodA(rMap) {
int retryCount = 0;
boolean success = false;
while (retryCount < MAX_TRY_COUNT && !success) {
try {
// 模拟一个可能会抛出异常的操作
riskyOperation();
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
// 这里可以选择在重试之前等待一段时间
Thread.sleep(10000); // 例如等待1秒
}
}
if (!success) {
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
}
}
// 一个可能会抛出异常的操作
def riskyOperation() throws Exception {
// 模拟一个有50%概率抛出异常的操作
if (Math.random() < 0.5) {
throw new Exception("Simulated exception");
}
}
}
@@ -0,0 +1,932 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.client.AggregateIterable
import org.bson.Document
import org.bson.types.ObjectId
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
*/
class TownMiddleProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new TownMiddleProcess();
task.run()
}
def run(){
//创建省份
// createProvince()
//创建城市
// createCity()
//创建区域
// createRegion()
//创建乡镇街道
// createTown()
//修改乡镇街道别名
// updateTownAlias()
//中间表同步到100
// synchTown()
//添加地级市标志
// addCityFlag()
// addPinyin()
//打标记,将开发区与民政部相同的打上1
// addTjjDevFlag()
//打标记,用于标识乡镇街道是否输出及格式化输出街道
// addStreetFlag()
// addDistrictDiffFlag()
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
// updateCommunityDiffStreet()
// updateTownPoline()
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
}
def createProvince(){
def cat = "省"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
tjjMaps.each {key, val->
def udata = new BasicDBObject()
udata.put("province", key)
udata.put("tjj_province_code", val)
udata.put("mzb_province_code", mzbMaps.get(key))
udata.put("tjj_source_year",2023)
udata.put("mzb_source_year",2023)
lastRets << udata
}
if(diffKeys){
diffKeys.each {k->
def udata = new BasicDBObject()
udata.put("province", k)
if(tjjMaps.get(k)){
udata.put("tjj_province_code", tjjMaps.get(k))
udata.put("tjj_source_year",2023)
}
if(mzbMaps.get(k)){
udata.put("mzb_province_code", mzbMaps.get(k))
udata.put("mzb_source_year",2023)
}
lastRets << udata
}
}
saveToDB(lastRets, "省份")
}
def createCity(){
def cat = "城市"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def sameKeys = tjjKeys.intersect(mzbKeys)
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
sameKeys.each {k->
def v = tjjMaps.get(k)
lastRets << getAdministrativeCodeInfo(k, v, mzbMaps)
}
if(diffKeys){
diffKeys.each {k->
def v = tjjMaps.get(k)
lastRets <<getAdministrativeCodeInfo(k, v, mzbMaps)
}
}
//添加别名
lastRets.each {o->
def city = RegionData.formatCity(o.city)
if(city){
def cityObj = CommonDatautils.findAlCityDetail(city)
o.put("alias", cityObj.alias)
}else{
println o.province+" "+o.city +" 房价网无此城市"
}
}
//添加城市围栏
def cityPolylineMap = cityPolyline()
//添加pid
def provPidMap = createPidData("省份")
lastRets.each{o->
def shapeData = cityPolylineMap.get(o.city)
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
}
def provId = provPidMap.get(o.province)
if(provId){
o.pid = provId
}
}
saveToDB(lastRets, cat)
}
def createRegion(){
def cat = "区域"
def tjjMaps = findAllProcincesFromTJJ(cat)
def mzbMaps = findAllProcincesFromMZB(cat)
def tjjKeys = tjjMaps.keySet()
def mzbKeys = mzbMaps.keySet()
def sameKeys = tjjKeys.intersect(mzbKeys)
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
def lastRets = []
sameKeys.each {k->
def v = tjjMaps.get(k)
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
// ud.source = "tjj|mzb"
lastRets << ud
}
if(diffKeys){
diffKeys.each {k->
def v = tjjMaps.get(k)
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
// if(v){
// ud.source = "tjj"
// }else{
// ud.source = "mzb"
// }
lastRets << ud
}
}
//添加别名
def regionMap = CommonDatautils.getAllCityFullRegions()
lastRets.each {o->
if(o.district.contains("开发区")){
println "ssss"
}
def city = RegionData.formatCity(o.city)
def district = RegionData.formatRegion(city, o.district)
if(city && district){
def cMap = regionMap.get(city)
if(cMap){
def regionInfoMap = cMap.get(district)
if(regionInfoMap){
def regions = regionInfoMap.get("pattern")
o.put("alias", regions)
o.put("mzb_district_name", regionInfoMap.full_region)
if(!o.mzb_district_code && regionInfoMap.full_region){
def mzb_val_str = mzbMaps.get(o.province+"|@|"+o.city+"|@|"+regionInfoMap.full_region)
if(mzb_val_str){
def mzb_vals = mzb_val_str.split("\\|@\\|")
o.put("mzb_province_code", mzb_vals[0])
o.put("mzb_city_code", mzb_vals[1])
o.put("mzb_district_code", mzb_vals[2])
}
}
}
}
}else{
println o.province+" "+o.city+" "+ o.district +" 房价网无此城市"
}
}
//添加pid
def provPidMap = createPidData("城市")
//添加区域围栏
def regionPolylineMap = regionPolyline()
lastRets.each{o->
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
if(shapeData){
o.shape = shapeData.shape
o.location = shapeData.location
}
def provId = provPidMap.get(o.city)
if(provId){
o.pid = provId
}
}
//是否为县级市
saveToDB(lastRets, cat)
}
def createTown(){
def cat = "乡镇街道"
def tjjMaps = findAllProcincesFromTJJ(cat)
def lastRets = []
def govMap = findAllGovTown()
tjjMaps.each {k,v->
def ud = getAdministrativeCodeInfo(k, v, null)
if(govMap && govMap.containsKey(k)){
ud.put("has_gov", govMap.get(k))
}
lastRets << ud
}
//添加区域围栏
def regionPolylineMap = townPolyline()
//添加pid
def provPidMap = createPidData("区域")
lastRets.each{o->
def shapeData = getStreetPolyline(regionPolylineMap, o, o.district)
if(!shapeData){
def newRegion = o.district
def standardRegion = RegionData.formatRegion(o.city,o.district)
if(standardRegion && standardRegion!=newRegion){
newRegion = standardRegion
shapeData = getStreetPolyline(regionPolylineMap, o, newRegion)
}
}
if(shapeData){
o.shape = shapeData.shape
if(shapeData.location){
o.location = shapeData.location
}
}
def provId = provPidMap.get(o.city+"@"+o.district)
if(provId){
o.pid = provId
}
}
//是否为县级市
saveToDB(lastRets, cat)
}
def getStreetPolyline(regionPolylineMap, o, district){
def shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+o.street)
if(!shapeData){
def streetKey = o.street
if(streetKey.toString().endsWith("镇")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"街道"
}else if(streetKey.toString().endsWith("乡")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇"
}else if(streetKey.toString().endsWith("街道")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"镇"
}else if(streetKey.toString().endsWith("地区")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("地区"))+"镇"
}
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
if(!shapeData){
streetKey = o.street
if(streetKey.toString().endsWith("镇")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"乡"
}else if(streetKey.toString().endsWith("乡")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇"
}else if(streetKey.toString().endsWith("街道")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"社区"
}
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
}
if(!shapeData){
streetKey = o.street
if(streetKey.toString().endsWith("镇")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"地区"
}else if(streetKey.toString().endsWith("街道")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"乡"
}
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
}
if(!shapeData){
if(streetKey.toString().endsWith("镇")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))
}else if(streetKey.toString().endsWith("乡")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))
}else if(streetKey.toString().endsWith("街道")){
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))
}
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
}
}
return shapeData
}
def getAdministrativeCodeInfo(key, val, mzbMaps){
def udata = new BasicDBObject()
def keys = key.split("\\|@\\|")
def vals = null
if(val){
vals = val.split("\\|@\\|")
}
def mzb_vals = null
if(mzbMaps && mzbMaps.containsKey(key)){
def mzb_val_str = mzbMaps.get(key)
mzb_vals = mzb_val_str.split("\\|@\\|")
}
if(keys.size()>=1){
udata.put("province", keys[0])
if(vals && vals.size()>=1){
udata.put("tjj_province_code", vals[0])
udata.put("tjj_source_year",2023)
}
if(mzb_vals && mzb_vals.size()>=1){
udata.put("mzb_province_code", mzb_vals[0])
udata.put("mzb_source_year",2023)
}
}
if(keys.size()>=2){
udata.put("city", keys[1])
if(vals && vals.size()>=2){
udata.put("tjj_city_code", vals[1])
}
if(mzb_vals && mzb_vals.size()>=2){
udata.put("mzb_city_code", mzb_vals[1])
}
}
if(keys.size()>=3){
udata.put("district", keys[2])
if(vals && vals.size()>=3) {
udata.put("tjj_district_code", vals[2])
}
if(mzb_vals && mzb_vals.size()>=3){
udata.put("mzb_district_code", mzb_vals[2])
}
}
if(keys.size()>=4){
udata.put("street", keys[3])
if(vals && vals.size()>=4) {
udata.put("tjj_street_code", vals[3])
}
if(mzb_vals && mzb_vals.size()>=4){
udata.put("mzb_street_code", mzb_vals[3])
}
}
return udata
}
def updateTownAlias(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇别名问题审核交付_0418.xlsx",{ row->
if(row['G'] && row['G']=="不处理"){
def q = new BasicDBObject()
q.put("city",row['C'])
q.put("district",row['D'])
q.put("street",row['E'])
def d = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(d){
def alias = []
alias << row['F']
def udata = new BasicDBObject()
udata.put("alias", alias)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}else{
println row['C']+" "+row['D']+" "+row["E"]
}
}
})
}
def synchTown(){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000)
def datas = []
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
datas.clear();
}
}
def saveToDB(lastRets, type){
def dataList = []
for(def ret : lastRets){
ret.put("c_date", new Date())
ret.put("m_date", new Date())
ret.put("type", type)
dataList << ret
if (dataList.size() >= 5000) {
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
dataList.clear();
}
}
if (dataList.size() > 0) {
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
dataList.clear();
}
}
def createPidData(def type){
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type)).batchSize(5000)
def map = [:]
def pids = []
def step = 2
if(type == "城市"){
step = 4
}else if(type == "区域"){
step = 6
}
for(def d : dbcursor){
def pid = null
def code = null
if(d.mzb_province_code){
if(type == "省份") {
code = d.mzb_province_code.toString()
}else if(type == "城市"){
code = d.mzb_city_code.toString()
}else if(type == "区域"){
code = d.mzb_district_code.toString()
}
}else if(d.tjj_province_code){
if(type == "省份") {
code = d.tjj_province_code.toString()
}else if(type == "城市"){
code = d.tjj_city_code.toString()
}else if(type == "区域"){
code = d.tjj_district_code.toString()
}
}
pid = code.substring(0,step)
if(pids.contains(d.city+"-"+pid)){
pid = code
}
if(!pids.contains(d.city+"-"+pid)){
pids << d.city+"-"+pid
}
if(type == "省份"){
map.put(d.province, pid)
}else if(type == "城市"){
map.put(d.city, pid)
}else if(type == "区域"){
map.put(d.city+"@"+d.district, pid)
}
}
return map;
}
def findAllGovTown(){
def q = new BasicDBObject();
q.put("has_gov", new BasicDBObject('$ne', ''))
def keys = new BasicDBObject()
keys.put("province_name",1)
keys.put("city_name",1)
keys.put("area_name", 1)
keys.put("street_name", 1)
keys.put("has_gov", 1)
def dbcursor = COMMUNITY_CONN.find(q,keys).batchSize(5000)
def map = [:]
for(def d : dbcursor){
def key = d.province_name+"|@|"+d.city_name+"|@|"+d.area_name+"|@|"+d.street_name
if(!map.containsKey(key)){
map.put(key, d.has_gov)
}
}
return map;
}
def findAllProcincesFromTJJ(cat){
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
def match = null
if(cat=="省"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code'))
}else if(cat=="城市"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code'))
}else if(cat=="区域"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code'))
}else if(cat=="乡镇街道"){
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("street_name", '$street_name').append("street_code", '$street_code'))
}
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city_name").toString()!="省直辖县级行政区划"){
def key = obj.get("province_name")
def value = obj.get("province_code")
if(cat=="城市"){
key = key +"|@|"+ obj.get("city_name")
value = value +"|@|"+ obj.get("city_code")
}else if(cat=="区域"){
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")
}else if(cat=="乡镇街道"){
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("street_name")
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("street_code")
}
int count = (Integer) result.get("count");
map.put(key, value)
}
}
return map;
}
def findAllProcincesFromMZB(cat){
BasicDBObject match = null;
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
DBObject groupFields = new BasicDBObject();
if(cat == "省"){
match = new BasicDBObject('$match', new BasicDBObject("level", "省份"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code'))
}else if(cat == "城市"){
match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code'))
}else if(cat == "区域"){
match = new BasicDBObject('$match', new BasicDBObject("level", "区域"));
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code').append("district",'$district').append("mca_district_code",'$mca_district_code'))
}
groupFields.put("count", new BasicDBObject('$sum', 1));
DBObject group = new BasicDBObject('$group', groupFields);
// 执行聚合查询
AggregationOutput output = ADMINISTRATIVE_DIVISION_CONN.aggregate(Arrays.asList(match, group));
def map = [:]
// 遍历结果
for (DBObject result : output.results()) {
def obj = (DBObject ) result.get("_id");
if(obj.get("city").toString()!="省直辖县级行政区划"){
def key = obj.get("province")
def value = obj.get("mca_province_code")
if(cat == "城市"){
key = key +"|@|"+obj.get("city")
value = value + "|@|" + obj.get("mca_city_code")
}else if(cat == "区域"){
key = key +"|@|"+obj.get("city")+"|@|"+obj.get("district")
value = value + "|@|" + obj.get("mca_city_code")+ "|@|" + obj.get("mca_district_code")
}
int count = (Integer) result.get("count");
map.put(key, value)
}
}
return map;
}
def cityPolyline(){
def dbcursor = CITYPOLYLINE_COLL.find(new BasicDBObject())
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.name+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
map.put(d.name, udata)
}
return map;
}
def regionPolyline(){
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
def fullCity = fullCityMap.get(d.city)
map.put(fullCity.city+"|@|"+d.region, udata)
}
return map;
}
def townPolyline(){
def dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject()).batchSize(5000)
def map = [:]
for(def d : dbcursor){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def center = d.center
if(center){
def pointArrs = center.split(",")
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
}
map.put(d.city+"|@|"+d.fjw_region+"|@|"+d.town, udata)
}
return map;
}
def addCityFlag(){
// ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row->
// if(row['A']){
// def q = new BasicDBObject("type", "城市")
// q.put("city", row['A'])
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
// if(obj){
// def udata = new BasicDBObject()
// udata.put("prefecture_level_city", "是")
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
// }else{
// println row['A']+" 没找到该城市"
// }
// }
// });
ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row->
if(row['A']){
def q = new BasicDBObject("type", "区域")
q.put("district", row['A'])
def count = ADMINISTRATIVE_MAPPING_COLL.count(q)
if(count && count == 1){
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
def udata = new BasicDBObject()
udata.put("county_level_city", "是")
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
}else{
println row['A']+" 没找到该城市"
}
}
},1);
}
def addPinyin(){
def q = new BasicDBObject("type", "乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
for(def d : dbcursor){
if(d.street){
def street_pinyin = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.street, " ")
def udata = new BasicDBObject()
udata.put("street_pinyin", street_pinyin)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
def addTjjDevFlag(){
def q = new BasicDBObject("type", "区域")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
for(def d : dbcursor){
if(d.district && d.mzb_district_name && d.district==d.mzb_district_name){
def udata = new BasicDBObject()
udata.put("district_diff", 1) //民政部,统计局 名称相同 district_diff打1
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
def addStreetFlag(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道-结果.xlsx",{ row->
if(row['I']){
if(row['I']=="标0"){
def udata = new BasicDBObject()
udata.put("street_output", 0) //0:不输出 1:输出
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
}else if(row['I']=="标1"){
def udata = new BasicDBObject()
udata.put("street_output", 1) //0:不输出 1:输出
if(row['G']){
udata.put("street", row['G']) //街道中去除区域
}
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
}
}
});
}
def addDistrictDiffFlag(){
ExcelUtils.modify("D:\\数据\\数据建设\\区域-结果.xlsx",{ row->
def flag = row['F'].toString().toDouble().toInteger()
def udata = new BasicDBObject()
udata.put("district_diff", flag)
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
});
}
def updateCommunityDiffStreet(){
def q = new BasicDBObject("type", "乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
for(def d : dbcursor){
def query = new BasicDBObject()
query.put("city_name", d.city)
query.put("area_name", d.district)
query.put("street_name", d.street)
def obj = COMMUNITY_CONN.findOne(query)
if(!obj){
query.put("street_name", d.district+d.street)
obj = COMMUNITY_CONN.findOne(query)
if(obj){
def newStreet = obj.street_name.replace(d.district, "")
def udata = new BasicDBObject()
udata.put("street_name", newStreet)
COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true)
println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}"
}else{
println d.city +" "+d.district+" "+d.street
}
}
}
}
def updateTownPoline(){
def city = "唐山市"
def name = "高新技术开发区街道"
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
def d = TOWNPOLYLINE_COLL.findOne(q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
println d.city+" "+d.region+" "+d.town+" "+rets.size()
def udata = new BasicDBObject()
udata.put("shape", resultMap)
def updateQuery = new BasicDBObject("city", city).append("street",name)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
def checkTown(){
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def ad : dbcursor){
def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street)
def count = TOWNPOLYLINE_COLL.count(polyline_q)
if(count==1){
println ad.city+" "+ ad.street+" "+count
def d = TOWNPOLYLINE_COLL.findOne(polyline_q)
if(d){
def resultMap = [:]
resultMap.type = "polygon"
def arrs = d.polyline.split("\\|")
def rets = []
for(def s : arrs){
def points = s.split(";")
def pointList = []
for(def pointStr : points){
def pointArr = pointStr.split(",")
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
}
rets << pointList
}
resultMap.coordinates = rets
if(rets.size()>1) {
resultMap.type = "MultiPolygon"
}
def udata = new BasicDBObject()
udata.put("shape", resultMap)
udata.put("polyline_source", "区域+街道匹配")
def updateQuery = new BasicDBObject("_id", ad._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
}
}
def updateTownshipByRegion(){
def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域")
def keys = new BasicDBObject()
keys.put("mzb_district_name",1)
keys.put("mzb_district_code",1)
keys.put("city",1)
keys.put("district",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
for(def d : dbcursor){
def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city)
def udata = new BasicDBObject()
udata.put("mzb_district_name",d.mzb_district_name)
udata.put("mzb_district_code",d.mzb_district_code)
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
}
}
}
@@ -0,0 +1,324 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class TownshipStrandUpdate {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
def fullCityMap = null
def regionMap = null
def commMap = null
def polylineMap = null
def newAdvMappingMap = null
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new TownshipStrandUpdate()
task.process()
}
void process() {
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
polylineMap = CommonDatautils.findAllTownRanges()
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
citys = citys.reverse()
for(def city : citys){
if(["中山"].contains(city)){
continue
}
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists)
def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}")
}
}
}
def checkData(def dists){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
rets << running(d)
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return dists
}
def running(d){
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
def gaodeMap = null
def town = null
def mapTown = null
def oldSimlarTown = null
def newSimlarTown = null
if(d.lng){
d.lng = d.lng/10000000000.0
d.lat = d.lat/10000000000.0
if(d.township){
town = getSimilarTown(regions, fullCity, d.township)
}
if(town && town.shape && town.shape.coordinates){
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
if(flag){
d.polygon_town = town.street
}
}
d.strand_street = ""
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
if(mapTown){
d.strand_street = mapTown.street
if(mapTown.shape && mapTown.shape.coordinates){
d.map_town_shape = "有"
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
if(flag){
d.map_polygon_town = mapTown.street
}
}
}else{
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
if(oldSimlarTown){
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
d.old_similar_town_score = oldSimlarTown.score
}
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
if(newSimlarTown){
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
d.new_similar_town_score = newSimlarTown.score
}
}
}
}
def mapCity = null
def mapRegion = null
def mapTownship = null
def mapAddr = null
if(gaodeMap){
if(gaodeMap.city && gaodeMap.city!="[]"){
mapCity = gaodeMap.city
}
if( gaodeMap.district && gaodeMap.district!='[]'){
mapRegion = gaodeMap.district
}
if(gaodeMap.township && gaodeMap.township!='[]'){
mapTownship = gaodeMap.township
}
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
mapAddr = gaodeMap.format_addr
}
}
d.mapCity = mapCity
d.mapRegion = mapRegion
d.mapTownship = mapTownship
d.mapAddr = mapAddr
if(mapCity && mapRegion){
if(mapCity == fullCity.city){
if(regions.contains(mapRegion)){
if(mapTownship){
if(town){
d.old_tjj_township = "是"
}else{
d.old_tjj_township = "否"
}
if(mapTown){
d.new_tjj_township = "是"
}else{
d.new_tjj_township = "否"
}
if(mapTownship == d.township){
if(town){
d.remark = "相同且为统计局乡镇"
}else{
d.remark = "非统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}else{
if(mapTown){
d.remark = "更正"
}else{
d.remark = "不同且不为统计局乡镇"
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
}
}
}else{
d.remark = "地图无乡镇街道"
d.opt = "地图无街道信息"
}
}else{
d.remark = "区域不同"
d.opt = "经纬度可能跨区域"
}
}else{
d.remark = "城市不同"
d.opt = "经纬度可能跨城市"
}
}
if(d.map_town_shape && d.map_town_shape=="有"){
if(!d.map_polygon_town){
d.opt = "经纬度可疑"
}
}
return d;
}
def getSimilarTownByStr(fullCity, townstr, townList){
def maxScore = 0
def town = null
for(def townObj : townList){
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
if(score > maxScore){
maxScore = score
town = townObj
}
}
if(town){
town.score = maxScore
}
return town
}
def getSimilarTown(regions, fullCity, townstr){
def town = null
for(def r : regions){
def townList = commMap.get(fullCity.city+"#"+r)
for(def townObj : townList){
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
town = townObj
}else if(townObj.alias && townObj.alias.contains(townstr)){
town = townObj
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
if(townstr.contains(newTown)){
town = townObj
}
}
if(town){
break
}
}
}
return town
}
def getFullRegion(fullCity, d){
def regions = regionMap.get(fullCity.city)
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.region)){
names << r.district
}
}
}
return names;
}
def createDatas(dists){
def rets = []
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
for(def d : dists){
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
}
return rets;
}
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
if(addressComponent.city.size()>0){
city = addressComponent.city
}else{
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
city = addressComponent.district
}else{
city = addressComponent.province
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
if(district || count>10){
stop = true
}else{
count ++
Thread.sleep(3000)
print "."
}
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
}
@@ -1,13 +1,46 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class TradeDatasDistBatch {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
static main(args) {
// def
def task = new TradeDatasDistBatch()
task.run()
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
def udata = new BasicDBObject()
udata.put("city", d['A'])
udata.put("region", d['B'])
udata.put("name", d['C'])
udata.put("alias", d['D'])
alias_fang_COLL.save(udata)
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def flag = null
// if(!dist){
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
// if(rets && rets.size()==1){
// flag = "别名匹配"
// dist = rets[0]
// }
// }else{
// flag = "名称匹配"
// }
// if(flag){
// d['F'] = dist.city
// d['G'] = dist.region
// d['H'] = dist.name
// }
})
}
}
@@ -135,6 +135,26 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def format(def city, def address){
def jsonObj = null
def content = null
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
}
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
@@ -228,9 +248,9 @@ class FangjiaClientUtils {
def url = null;
try {
address = address.replace("{","").replace("\\","")
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -334,13 +354,13 @@ class FangjiaClientUtils {
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
// println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
@@ -0,0 +1,138 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 房价网内部模型估值项目
* 小区建筑类型数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataCommbuildtypeBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rets = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def distIds = FjwPgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
def rows = createDatas(cityCodeDatas.get(city), distIds)
rets.addAll(rows)
println city +" "+rows.size()
}
delPgData()
saveToPgSql(rets)
}
def createDatas(cityCode, dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def id = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dist = CommonDatautils.findDistrictFieldById(id, ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
if(dist){
def ret = createData(cityCode, dist)
if(ret){
rets.addAll(ret)
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(cityCode, dist){
def types = CcbCodeUtils.parseBuildType(dist.build_type)
def rets = []
for(def type : types){
def ret = []
ret << cityCode
ret << dist._id.toString()
ret << type
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
rets << ret
}
return rets;
}
def delPgData(){
sqlInstance.execute("delete from base.bigdata_commbuildtype")
println "base.bigdata_commbuildtype 数据已删除"
}
def saveToPgSql(rows){
def sqls = []
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
def sql = "insert into base.bigdata_commbuildtype values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
println "start: "+new Date()
def index =0
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
index++
println index
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,370 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 房价网内部估值项目使用
* 小区数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
}
if(today > 10){
delPgData()
saveToPgSql(rows)
}
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
if(!fangjiaDists.contains(ccbId)){
def datas = ccbMaps.get(ccbId)
datas[datas.size()-1] = 1
rets << datas
}
}
println "添加删除状态小区:"+rets.size()
return rets
}
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
//def rets = Collections.synchronizedList(new ArrayList());
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDataMap, distCodeMap, dist)
if(ret){
rets << ret
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def createData(def cityCodeDataMap, def distCodeMap, def dist){
def ret = []
try{
def data = distCodeMap.get(dist.region)
if(!data){
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
return ret;
}
ret << cityCodeDataMap.get(dist.city)
if(data){
ret << data.district_code
}else{
ret << ""
}
ret << dist._id.toString()
if(data){
ret << data.district
}else{
ret << ""
}
ret << dist.name
ret << dist.address
if(dist.lat){
def lat = dist.lat/10000000000d
def lng = dist.lng/10000000000d
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << dist.all_area
ret << dist.build_area
ret << dist.virescence_percent
ret << dist.cubage_percent
ret << dist.car_site_count
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
ret << dist.household_count
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger()
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "pa101"
ret << ""
if(dist.address){
if(dist.street){
ret << dist.street
if(dist.street_number){
ret << dist.street_number
}else{
ret << ""
}
}else{
ret << ""
ret << ""
// def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
// if(jsonObj && jsonObj.result && jsonObj.result.street){
// if(jsonObj.result.street){
// ret << jsonObj.result.street
// }else{
// ret << ""
// }
//
// if(jsonObj.result.streetNumber){
// ret << jsonObj.result.streetNumber
// }else{
// ret << ""
// }
// }else{
// ret << ""
// ret << ""
// }
}
}else{
ret << ""
ret << ""
}
ret << 0
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << ""
ret << ""
ret << ""
ret << 0
}catch(Exception e){
println dist.city + " "+dist.name+" error"
e.printStackTrace()
}
return ret;
}
public def findPgCcbDistrict(cityid){
def sql = "select * from base.bigdata_community where urbn_id='${cityid}'";
def map = [:]
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_dtl_adr
data << r.gujia_bds_ltt
data << r.gujia_bds_lgt
data << r.gujiabdsocpoldtotarea
data << r.gujiabds_bld_tot_area
data << r.gujia_bds_grnrate
data << r.gujia_bds_vol_rate
data << r.gujia_bds_park_num
data << r.gujia_bds_bld_yr_rc
data << r.gujia_bds_hs_use_val
data << r.totunits
data << r.gujia_bds_data_tm_val
data << r.gujiabds_udt_flag_val
data << r.data_udt_tm
data << r.gujia_comm_tp_val
data << r.gujia_comm_fmnm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujiawthr_new_pan_ind
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.gujia_src_co
data << r.gujia_vlgtwn_str_mng
data << r.gujia_lgc_admndiv_cd
data << r.gujia_lgc_adiv
data << r.gujia_bds_bds_st
map.put(r.gujia_comm_id, data)
});
return map
}
def delPgData(){
sqlInstance.execute("delete from base.bigdata_community")
println "base.bigdata_community 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_community values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
sqlInstance.withBatch(50000, sql){ ps->
for(def r : rows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def saveToPgSql2(rows){
def newRows = []
for(def r : rows) {
def newR = []
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
// r[i] = null
newR << null
}else{
newR << r[i]
}
}
newR << null
newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
newR << null
newR << '2999-12-31'
newR << null
newR << null
newR << null
newRows << newR
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def index = 0
sqlInstance.withBatch(50000, sql){ ps->
for(def r : newRows) {
index++
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
index
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("\\|@\\|")
rows << row
}
return rows;
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
sqlInstance.executeUpdate(sql)
}
}
}
@@ -0,0 +1,786 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 成交数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataDealBatch extends BaseBatch{
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwBigdataDealBatch(), args);
}
@Override
public void process(Object args) {
def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args)
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
def trades = getCityData(city, statMap, ppeDists)
if(trades){
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
rows.addAll(rets)
println "【${index}】"+city +" 成交共:"+rets.size()
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rows)
}
}
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
// saveToPgSql2(datas)
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("\\|@\\|")
rows << row
}
return rows;
}
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
def result = Collections.synchronizedList(new ArrayList());
dealIds = Collections.synchronizedList(dealIds);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
// def lianjiaPriceMap = findThirdPartyPrice("上海")
// def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
def ccbPriceMap = null
if(city == "上海"){
ccbPriceMap = getCommunityPrices("上海")
}
for(def i=0;i<rets.size();i++){
final def trade = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(dealIds.contains(trade._id.toString())){
println "重复:"+trade._id.toString() + " "+ trade.city+ " " +trade.district_name
}else{
if(source == "youda"){
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}
}else{
def ret = converData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println "查询完成......."
latch.await(5, TimeUnit.MINUTES);
return result
}
def findThirdPartyPrice(def city) {
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for (def db : dbcursor) {
def price = null
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
price = db.lianjia_esf_price
}
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
price = db.beike_esf_price
}
if(price){
map.put(db.region+"##"+db.name, price)
}
}
return map
}
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def ccbPriceMap){
def ret = []
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
def price = ccbPriceMap.get(obj._id.toString())
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=20){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
def street = ""
def streetNumber = ""
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}
if(!street){
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
streetNumber = number
}
}else{
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
if(!street){
if(obj.address){
if(obj.address.toString().contains("弄")){
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
if(number){
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
streetNumber = number
}
}else{
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
if(jsonObj && jsonObj.searchAddress){
street = jsonObj.searchAddress.street
streetNumber = jsonObj.searchAddress.streetNumber
}
}
}
}
ret << street
ret << streetNumber
if(obj){
ret << obj._id.toString()
ret << trade.fjName
}
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
ret << ""//期数
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
if(roomNum && roomNum.contains("0")){
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
}else{
ret << trade.floor
}
ret << ""
ret << trade.height
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << "99"
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
}
}
return ret;
}
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def data = distCodeMap.get(trade.region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << CcbCommonUtils.getCcbCity(trade.city)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
if(trade.address){
if(trade.address.toString().contains("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
if(number){
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
ret << number
}else{
ret << ""
ret << ""
}
}else{
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
}else{
ret << ""
ret << ""
}
}
}else{
ret << ""
ret << ""
}
ret << trade.dist_id.toString()
ret << trade.master_name
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
ret << ""//期数
ret << trade.floor
ret << ""
ret << trade.height
// ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
def arrs = CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
def buildType = ''
if(arrs){
buildType = arrs[0]
}
ret << buildType
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
ret << trade.area
ret << ""
ret << getHouseRoom(buildType, trade.room, trade.hall)
ret << trade.room
ret << trade.hall
ret << ""
ret << trade.toilet
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
ret << ""
ret << CcbCodeUtils.getDirectionCode(trade.direction)
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.avg_price
ret << trade.total_price
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
ret << "02"
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
ret << ""
def root = sourceMap.get(trade.source)
if(root){
if(CcbCommonUtils.rootMap.get(root)){
ret << CcbCommonUtils.rootMap.get(root)
}else{
ret << "99"
}
}else{
ret << ""
}
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.total_price
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
def getHouseRoom(buildtype, def roomcnt, def livingroomcnt){
def housetype = null
if(buildtype && ["0112","0110","0109"].contains(buildtype)){
housetype = "07"
}else if(buildtype && ["0107","0106","0105"].contains(buildtype)){
housetype = "16"
}else{
if(roomcnt && roomcnt==4){
if(livingroomcnt>1){
housetype = "14"
}else{
housetype = "11"
}
}else if(roomcnt && roomcnt==3){
if(livingroomcnt>1){
housetype = "13"
}else{
housetype = "10"
}
}else if(roomcnt && roomcnt==2){
if(livingroomcnt>1){
housetype = "12"
}else{
housetype = "09"
}
}else if(roomcnt && roomcnt==1){
housetype = "08"
}else{
housetype="99"
}
}
return housetype;
}
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all")
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null
def endPrice = null
// if(price){
// if(dist.city=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else if(dist.city=="杭州"){
// startPrice = (int)(price - price*0.15)
// endPrice = (int)(price*0.15 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// }
// def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
// if(!trades && dist.address){
// def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
// if(flag){
// trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
//
// if(!trades){
// trades = CcbCommonUtils.findTradesDetailByAddressAlias(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
// }
// }
//
// }
def trades = CcbCommonUtils.findTradesDetailById(dist, startDate, endDate)
if(trades){
// def unitPrice= ppeDistsMap.get(dist._id.toString())
// if(unitPrice){
// for(def t : trades){
// t.avg_price = (int)(t.avg_price*0.03+t.avg_price)
// }
// }
tradesList.addAll(trades)
}
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return tradesList
}
def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
q.put("area", new BasicDBObject('$gte', 20))
// q.put("status", 0)
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
def rets = []
def index = 0
for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){
rets << obj;
// def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
// if(price){
// def startPrice = 0
// def endPrice= 0
// if(obj.fjCity=="上海"){
// startPrice = (int)(price - price*0.1)
// endPrice = (int)(price*0.5 + price)
// }else{
// startPrice = (int)(price - price*0.5)
// endPrice = (int)(price*0.5 + price)
// }
// if(obj.price>=startPrice && obj.price<=endPrice){
// rets << obj;
// }else{
// index++
// println index + " "+obj
// }
// }
}
}
return rets;
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", new BasicDBObject('$gte', weekend))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
def map = new ConcurrentHashMap()
for(def d : dbcursor){
def price = map.get(d.region+"##"+d.district_name)
if(!price && d.value_type==2){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
}
return map
}
def findHistoryDealIds(def city){
def sql = "SELECT gujia_bds_mdl_id from base.bigdata_deal where urbn_nm='${city}'"
def ids = []
try{
pgsqlInstance.eachRow(sql,{
ids << it.gujia_bds_mdl_id
});
}catch(Exception e){
}
return ids
}
def addDealPriceMatch(city){
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd")
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
final CountDownLatch latch = new CountDownLatch(trades.size())
def final AtomicInteger count_ = new AtomicInteger(0)
for(def i=0;i<trades.size();i++){
final def t = trades[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
count_.addAndGet(1)
try{
if(!t.match_id){
def region = RegionData.formatRegion(t.city, t.region)
def rMap = null
if(region) {
if(t.house_use && t.house_use.contains("商住")){
if(!t.name.contains("商住")){
t.name = t.name +"商住楼"
}
}
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
if(!rMap && t.address){
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
if(flag){
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
}
}
if(rMap && rMap.region==region){
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
}else {
if (rMap) {
println "区域不同"
}
}
}
println count_.get()+" "+t+" "+ rMap
}
}catch(Exception e){
e.printStackTrace()
}finally {
latch.countDown()
}
}
}
);
}
latch.await()
println city +" "+trades.size()+" "+new Date().toString()
}
def delPgData(){
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
pgsqlInstance.execute("delete from base.bigdata_deal where p9_start_date='${month}'")
println "base.bigdata_deal 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_deal values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
}
}
println "end: "+new Date()
}
def saveToPgSql2(rows){
def newRows = []
for(def r : rows) {
def newR = []
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
// r[i] = null
newR << null
}else{
newR << r[i]
}
}
newR << null
newR << "2023-09-28"
newR << null
newR << '2999-12-31'
newR << null
newR << null
newR << null
newRows << newR
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : newRows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,968 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.google.common.hash.*
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.nio.charset.StandardCharsets
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* 建行模型估值项目
* 出售房源数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwBigdataForsaleBatch extends BaseBatch{
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
}
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200)
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
static main(args) {
runBatch(new FjwBigdataForsaleBatch(), args);
}
@Override
public void process(Object args) {
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def startDate = lastMonth+"18"
def endDate = curMonth+"19" //正式运行时curMonth要改
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
for(def city : citys){
index++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
}
}
def statCity(def citys){
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1)
def houseCitys = []
for(def d : datas){
houseCitys << d['C']
}
for(def city : citys){
if(!houseCitys.contains(city)){
def scity = CcbCommonUtils.specialCityMap.get(city)
if(!scity){
println city
}
}
}
}
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
println city + "(原)房源总量:"+retMap.size()
def allDatas = retMap.values()
saveToPgSql(allDatas)
}
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
def retMap = new ConcurrentHashMap()
for(def i=0;i<districts.size();i++){
final def rMap = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap)
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
// }
if(ret){
def areaData = distCodeMap.get(rMap.region)
if(areaData){
def regionCode = areaData.district_code
def cityCode = cityCodeDataMap.get(city)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
curhousefilter.put(m.houseId)
houseIdsfilter.put(m.houseId)
}else{
println city +" "+m.houseId+" 已存在"
}
}
}else{
println city +" "+rMap.region+" 没有找到对应的区域"
}
}
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
} catch (Exception e) {
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return retMap
}
def converData(def m){
def data = []
data << m.city_code
data << m.houseId
data << ""
data << m.city
data << m.region_code
data << m.region
data << m.district_id
data << m.district_name
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.address==null?"":m.address==null
data << m.build_year==null?"":m.build_year
data << m.proptype==null?"":m.proptype
data << m.propyears==null?"":m.propyears
data << m.avg_price==null?"":m.avg_price
data << m.total_price==null?"":m.total_price
data << ""
data << m.floor==null?"":m.floor
data << m.height==null?"":m.height
data << m.estate_type==null?"":m.estate_type
data << m.room_structure_code==null?"":m.room_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
data << ""
data << m.house_type==null?"":m.house_type
data << m.room==null?"":m.room
data << m.hall==null?"":m.hall
data << ""
data << m.toilet==null?"":m.toilet
data << m.direction_code==null?"":m.direction_code
data << m.fitment_code==null?"":m.fitment_code
data << ""
data << ""
data << m.p_date==null?"":m.p_date
data << ""
data << ""
data << m.root
data << ""
data << ""
data << ""
data << ""
data << ""
data << ""
data << m.c_date==null?"":m.c_date
data << m.m_date==null?"":m.m_date
data << m.source
data << ""
return data
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
rets1.addAll(rets2)
// if(rets1){
// def newRets = []
// def cityCode = cityCodeDataMap.get(city)
// def houseIds = getAllHouseIds(commId, cityCode)
// for(def r : rets1){
// if(!houseIds.contains(r.houseId)){
// newRets << r
// }else{
// println r.houseId +" 已存在"
// }
// }
// return newRets
// }
// return [];
return rets1;
}
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def rets = []
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
// if(houseIds.contains(seawater.id)){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
// def count = isExistFromHouse(seawater.id)
// if(count){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
//用以下方法会查出发布时间很久以前的
// QueryBuilder dateQueryBuilder = QueryBuilders.boolQuery()
// dateQueryBuilder.should(date_range)
// dateQueryBuilder.should(date_range_mdate)
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
// .must(area_range).must(dateQueryBuilder)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
def index = new Random().nextInt(6);
def rets = []
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
if(houses){
def ids = new HashSet()
for(def h : rets1){
ids << h.houseId
}
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(ids && ids.contains(seawater.id)){
continue;
}
if(estate_type2 && estate_type2=="普通住宅"){
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
continue;
}
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
if(vallaFlag){
continue;
}
}
if(!seawater.total_price){
continue;
}
if(!seawater.avg_price){
continue;
}
if(!seawater.height){
seawater.height = null
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
// if(!seawater.room){
// continue;
// }
if(!seawater.room){
seawater.room = null
}
if(seawater.room){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
if(!seawater.hall){
seawater.hall = null
}
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
continue;
}
if(!seawater.toilet){
seawater.toilet = null
}
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
continue;
}
def floor = parseFloor(seawater.floor_info)
if(seawater.height && floor && floor.isNumber()){
if(floor.toInteger()>seawater.height){
floor = null
seawater.height = null
}
}
if(!floor){
seawater.floor = null
}
if(floor && floor.toInteger()<1){
floor = null
}
def rowMap = [:];
if(cat == "rent"){
rowMap.rent_type = seawater.rent_type
if(seawater.avg_price && seawater.avg_price>1000){
continue
}
if(seawater.avg_price && seawater.avg_price<10){
continue
}
}
rowMap.houseId = seawater.id
rowMap.city = CcbCommonUtils.getCcbCity(city)
//rowMap.region =seawater.region
//rowMap.address = seawater.address
rowMap.area = seawater.area
rowMap.total_price = seawater.total_price
rowMap.avg_price = seawater.avg_price
rowMap.height = seawater.height=="null"?null:seawater.height
if(rowMap.height && rowMap.height<1){
rowMap.height = null
}
rowMap.floor = floor=="null"?null:floor
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
rowMap.root = "99"
}
def direction = parseDirection(seawater.direction)
rowMap.direction = direction=="null"?null:direction
def fitment = parseFitment(seawater.fitment)
rowMap.fitment = fitment=="null"?null:fitment
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
}
rets << rowMap
// if(houseIds.contains(seawater.id)){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
// def count = isExistFromHouse(seawater.id)
// if(count){
// println seawater.id +" 已存在"
// }else{
// rets << rowMap
// }
}
}
println name +" "+houses.size()+" last:"+rets.size()
return rets;
}
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) {
int retryCount = 0;
boolean success = false;
def rets = []
while (retryCount < MAX_TRY_COUNT && !success) {
try {
rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
// 如果操作成功,设置success为true
success = true;
} catch (Exception e) {
retryCount++;
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
// 这里可以选择在重试之前等待一段时间
Thread.sleep(3000); // 例如等待1秒
}
}
if (!success) {
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
}
return rets;
}
/**
* 是否是别墅
* @return
*/
public boolean isVilla(area, name, height, room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
def getPublicDate(def pDate){
if(pDate>=20181001 && pDate<20200201){
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
}
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
return pdate;
}
static def parseFloor(def floorDesc){
if(floorDesc){
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
if(floorDesc){
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
Matcher matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
matcher1 = pat1.matcher(floorDesc);
if (matcher1.find()) {
return matcher1.group(1)
}
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
// return "中层"
// }
//
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
// return "高层"
// }
//
//
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
// return "低层"
// }
if(floorDesc.isNumber()){
return floorDesc;
}
}
}
return null;
}
private String csvHandlerStr(String str) {
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
String tempDescription=str;
//如果有逗号
if(str.contains(",")||str.contains(",")){
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
if(str.contains("\"")){
// tempDescription=str.replace("\"", "\"\"");
tempDescription=str.replace("\"", "");
}
//在将逗号转义
tempDescription="\""+tempDescription+"\"";
}
return tempDescription;
}
static String parseFitment(String fitmentDesc) {
if (fitmentDesc == null) {
return null
}
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
if (fitmentDesc.equals("非毛坯")) {
return null
}
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
.entrySet().iterator()
Map.Entry<String, List<String>> entry
while (it.hasNext()) {
entry = it.next()
if (fitmentDesc.contains(entry.getKey())) {
return entry.getKey()
}
for (String fitmentKey : entry.getValue()) {
if (fitmentDesc.contains(fitmentKey)) {
return entry.getKey()
}
}
}
return null
}
static String parseDirection(String directionDesc) {
if (directionDesc == null) {
return null
}
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
StringBuffer sb = new StringBuffer()
for (int i = 0; i < directionDesc.length(); i++) {
String c = directionDesc.substring(i, i + 1)
if (DIRECTIONS.contains(c)) {
if (directionDesc.substring(i).startsWith(c + "#_#")) {
sb.append(c + " ")
} else {
sb.append(c)
}
}
}
for (String d : DIRECTIONS) {
if (sb.toString().startsWith(d)) {
return d
}
}
for (String d : DIRECTIONS) {
if (sb.toString().contains(d)) {
return d
}
}
return null
}
def marge(){
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
def ids = new HashSet();
def index = 0
file.eachLine{it->
index++
if(index>1){
def arrs = it.split("\\|@\\|")
def id = arrs[1]
if(!ids.contains(id)){
ids << id
}
}
}
println index
println ids.size()
}
def getAllHouseIds(def commid, def cityCode){
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'";
def houseIds = new HashSet()
sqlInstance.eachRow(sql,{it->
houseIds << it.FORSALEID
});
return houseIds;
}
def findAllHouseIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
filter.put(it.gujia_sell_id)
});
// pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale_2021",{it->
// filter.put(it.gujia_sell_id)
// });
return filter;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_forsale values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40],
r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,226 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import org.bson.types.ObjectId
/**
* 建行模型估值项目
* 人工指导价同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwGuiDingPriceBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwGuiDingPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01")
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
rows.addAll(rets)
println city + " " + rets.size()
}
}
saveToPgSql(rows)
}
def findFangDatas(def districts){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println " 小区量:"+districts.size()
def result = Collections.synchronizedList(new ArrayList());
for(def i=0;i<districts.size();i++){
final def dist = districts[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def obj = findMatchThirdPrice(dist)
if(obj && obj.avg_price){
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
return result
}
def findPersonDatas(def districts, def dataMap){
def result = []
for(def i=0;i<districts.size();i++) {
def dist = districts[i];
def price = dataMap.get(dist._id.toString())
if (price) {
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
println dist.city + " " + dist.region + " " + dist.name + " " + price
}
}
return result;
}
def findMatchThirdPrice(dist){
def q = new BasicDBObject()
q.put("match_city", dist.city)
q.put("match_region", dist.region)
q.put("match_name", dist.name)
q.put("s_date", sDate)
q.put("source","搜房")
def obj = dataProcessColl.findOne(q)
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
if(areaData){
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.get(dist.city)
ret << CcbCommonUtils.getCcbCity(dist.city)
ret << areaData.district_code
ret << areaData.district
ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
}
return rets;
}
def createDatas(cityCodeDatas, distCodeMap, datas){
def rets = []
if(datas) {
for (def ps : datas) {
def ret = []
def areaData = distCodeMap.get(ps.region)
if (areaData) {
ret << ps.row_id
ret << ps.dist_id
ret << ps.name
ret << cityCodeDatas.get(ps.city)
ret << ps.city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.avg_price
ret << "房价网"
ret << ""
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
}
def getDistrictMap(def city){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], "all");
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
}
return distMap;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.guided_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(300, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20])
}
}
println "end: "+new Date()
}
}
@@ -0,0 +1,39 @@
package com.fangjia.script.gj.fjwPriceBaseData
import groovy.sql.Sql
/**
* ·¿¼ÛÍøÄÚ²¿Ê¹ÓÃ
*/
class FjwPgsqlCommUtils {
public static def database = "fangjia"
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
public static def findAllDistrictIdsByOracle(def cityid){
def sql = "select gujia_comm_id from base.bigdata_community where urbn_id='${cityid}' and gujia_bds_bds_st=0";
def set = new HashSet()
sqlInstance.eachRow(sql,{r->
set << r.gujia_comm_id
});
return set
}
public static def findDistGovPrice(def city){
def ids = []
sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it->
ids << it.gujia_bds_id
});
return ids;
}
public static def deleteGuidedPrice(def dealdate){
def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'"
sqlInstance.executeUpdate(sql2)
}
}
@@ -0,0 +1,675 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.text.SimpleDateFormat
import java.util.concurrent.ConcurrentHashMap
/**
* 建行模型估值项目
* 第三方均价数据同步
*/
@BatchOwner(Watcher.GaoJuan)
class FjwThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304//
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
runBatch(new FjwThirdPartyPriceBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def stop = false
def month = -1
while(!stop){
if(sDate<202004){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)
}
month = month -1
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
if(sDate == 201912){
stop = true;
}
}
}
def runDatas(citys){
def rows = []
def index = 0
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
def gocIdsPrices = []
if(city == "深圳"){
gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city)
}
def statMap = getDistrictStatByCity(city, weekend)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all")
def thirdPriceMap = findThirdPartyPrice(city, statMap)
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices)
rows.addAll(rets)
println index+" "+city+" "+rets.size() +" "+rows.size()
}
saveToPgSql(rows)
}
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
def rets = []
for(def d : dists){
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
if(data){
def ret = []
ret << third.id.toString()
ret << d._id.toString()
ret << d.name
ret << cityCodeDatas.get(d.city)
ret << d.city
ret << data.district_code
ret << data.district
ret << sDate
if(third.zhugezhaofang_esf_price){
ret << third.zhugezhaofang_esf_price
}else{
ret << ""
}
if(third.anjuke_esf_price){
ret << third.anjuke_esf_price
}else{
ret << ""
}
if(third.soufang_esf_price){
ret << third.soufang_esf_price
}else{
ret << ""
}
if(third.lianjia_esf_price){
ret << third.lianjia_esf_price
}else{
ret << ""
}
if(third.cityhouse_esf_price){
ret << third.cityhouse_esf_price
}else{
ret << ""
}
if(third.maitian_esf_price){
ret << third.maitian_esf_price
}else{
ret << ""
}
if(third.beike_esf_price){
ret << third.beike_esf_price
}else{
ret << ""
}
if(third.get("365house_esf_price")){
ret << third.get("365house_esf_price")
}else{
ret << ""
}
if(third.zhongyuan_esf_price){
ret << third.zhongyuan_esf_price
}else{
ret << ""
}
if(third.tuboshi_esf_price){
ret << third.tuboshi_esf_price
}else{
ret << ""
}
if(third.qfang_esf_price){
ret << third.qfang_esf_price
}else{
ret << ""
}
if(third.fangtan_esf_price){
ret << third.fangtan_esf_price
}else{
ret << ""
}
if(third.fangduoduo_esf_price){
ret << third.fangduoduo_esf_price
}else{
ret << ""
}
if(third.get("5i5j_esf_price")){
ret << third.get("5i5j_esf_price")
}else{
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
return rets;
}
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){
def allRows = []
for(def d : dists){
if(gocPrices && gocPrices.contains(d._id.toString())){
continue;
}
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def data = distCodeMap.get(d.region)
if(data){
def dataMap = [:]
dataMap.COMMID = d._id.toString()
dataMap.COMMNAME = d.name
dataMap.CITYID = cityCodeDatas.get(d.city)
dataMap.CITYNAME = d.city
dataMap.DISTID = data.district_code
dataMap.DISTNAME = data.district
dataMap.YEAR_MONTH = sDate
def price_map = [:]
dataMap.price_map = price_map
if(third.zhugezhaofang_esf_price){
price_map.zhuge_avg_price = third.zhugezhaofang_esf_price
}
if(third.anjuke_esf_price){
price_map.anjuke_avg_price = third.anjuke_esf_price
}
if(third.soufang_esf_price){
price_map.fang_avg_price = third.soufang_esf_price
}
if(third.lianjia_esf_price){
price_map.lianjia_avg_price = third.lianjia_esf_price
}
if(third.cityhouse_esf_price){
price_map.cityhouse_avg_price = third.cityhouse_esf_price
}
if(third.maitian_esf_price){
price_map.maitian_avg_price = third.maitian_esf_price
}
if(third.beike_esf_price){
price_map.ke_avg_price = third.beike_esf_price
}
if(third.get("365house_esf_price")){
price_map.taofang_avg_price = third.get("365house_esf_price")
}
if(third.zhongyuan_esf_price){
price_map.zhongyuan_avg_price = third.zhongyuan_esf_price
}
if(third.tuboshi_esf_price){
price_map.tbs_avg_price = third.tuboshi_esf_price
}
if(third.qfang_esf_price){
price_map.qfang_avg_price = third.qfang_esf_price
}
if(third.fangtan_esf_price){
price_map.fangtan_avg_price = third.fangtan_esf_price
}
if(third.fangduoduo_esf_price){
price_map.fangduoduo_avg_price = third.fangduoduo_esf_price
}
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
}
}
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) {
def rets = []
for (def d : dists) {
def third = thirdPriceMap.get(d.region + "##" + d.name)
if (third) {
def data = distCodeMap.get(d.region)
if (data) {
third.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << cityCodeDatas.get(d.city)
ret << data.district_code
ret << d._id.toString()
ret << d.name
def web = ""
if(k=="zhugezhaofang_esf_price"){
web = "诸葛找房"
}else if(k=="anjuke_esf_price"){
web = "安居客"
}else if(k=="soufang_esf_price"){
web = "房天下"
}else if(k=="lianjia_esf_price"){
web = "链家"
}else if(k=="cityhouse_esf_price"){
web = "禧泰数据"
}else if(k=="maitian_esf_price"){
web = "麦田"
}else if(k=="beike_esf_price"){
web = "贝壳"
}else if(k=="365house_esf_price"){
web = "淘房365"
}else if(k=="zhongyuan_esf_price"){
web = "中原"
}else if(k=="tuboshi_esf_price"){
web = "兔博士"
}else if(k=="qfang_esf_price"){
ret << "Q房"
}else if(k=="fangtan_esf_price"){
ret << "房探"
}else if(k=="fangduoduo_esf_price"){
ret << "房多多"
}else if(k=="5i5j_esf_price"){
ret << "我爱我家"
}else{
ret << ""
}
ret << sourceMap.get(web)
ret << web
ret << sDate
ret << v
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
}
return rets;
}
def createData3(cityid){
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def dataMap = [:]
dataMap.COMMID = r.COMMID
dataMap.COMMNAME = r.COMMNAME
dataMap.CITYID = r.CITYID
dataMap.CITYNAME = r.CITYNAME
dataMap.DISTID = r.DISTID
dataMap.DISTNAME = r.DISTNAME
dataMap.YEAR_MONTH = r.YEAR_MONTH
def price_map = [:]
dataMap.price_map = price_map
if(r.ZHUGE_AVG_PRICE){
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
}
if(r.ANJUKE_AVG_PRICE){
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
}
if(r.FANG_AVG_PRICE){
price_map.fang_avg_price = r.FANG_AVG_PRICE
}
if(r.LIANJIA_AVG_PRICE){
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
}
if(r.CITYHOUSE_AVG_PRICE){
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
}
if(r.MAITIAN_AVG_PRICE){
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
}
if(r.KE_AVG_PRICE){
price_map.ke_avg_price = r.KE_AVG_PRICE
}
if(r.TAOFANG_AVG_PRICE){
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
}
if(r.ZHONGYUAN_AVG_PRICE){
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
}
if(r.TBS_AVG_PRICE){
price_map.tbs_avg_price = r.TBS_AVG_PRICE
}
if(r.QFANG_AVG_PRICE){
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
}
if(r.FANGTAN_AVG_PRICE){
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
}
if(r.FANGDUODUO_AVG_PRICE){
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
}
if(r.WIWJ_AVG_PRICE){
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
}
dataMap.updatetime = r.UPDATETIME
allRows << dataMap
});
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
dataMap.put("id", db._id.toString())
map.put(key, dataMap)
}
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
dataMap.put("soufang_esf_price", db.soufang_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
dataMap.put("qfang_esf_price", db.qfang_esf_price)
}
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
}
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
}
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
}
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
def price = statMap.get(key)
if(price){
def priceMap = [:]
def startPrice = (int)(price - price*0.3)
def endPrice = (int)(price*0.3 + price)
value.each{pk,pv->
if(pk.contains("price")){
if(pv>=startPrice && pv<=endPrice){
priceMap.put(pk,pv)
}
}else{
priceMap.put(pk,pv)
}
}
if(priceMap.size()>1){
resultMap.put(key, priceMap)
}
}
}else{
resultMap.put(key, value)
}
}
}
return resultMap
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"##"+d.district_name, d.avg_price)
}
return map
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(50000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18])
}
}
println "end: "+new Date()
}
}
@@ -13,7 +13,7 @@ class MsbPocTest {
task.run();
}
def run(){
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
// standard(inputfile)
// match(inputfile)
@@ -24,7 +24,7 @@ class MsbPocTest {
// cityPrice()
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx")
addCCBPrice(inputfile)
}
def cityPrice(){
@@ -69,21 +69,21 @@ class MsbPocTest {
def matchOld(path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['S'] && row['D']){
if(!row['Y'] && row['G']){
def city = row['C']
def region = row['D']
def name = row['F']
def address = row['H']
def region = null //row['D']
def name = row['E']
def address = row['G']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
if(jsonObject != null && jsonObject.precise!=-1) {
row['S'] = jsonObject.get("id")
row['T'] = jsonObject.get("city")
row['U'] = jsonObject.get("district")
row['V'] = jsonObject.get("name")
row['W'] = jsonObject.get("propertyType")
row['X'] = jsonObject.get("credit")
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
row['AC'] = jsonObject.get("propertyType")
// row['X'] = jsonObject.get("credit")
}
}
})
@@ -111,23 +111,18 @@ class MsbPocTest {
def match(file){
ExcelUtils.multiThreadingModify(file, { row ->
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
if(!row['Y']){
def city = row['C']
def region = row['D']
def region = null //row['B']
def address = row['G'] //row['G']//row['F']
println "[match] city:"+city+" region:"+region+" address:"+address
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
if(jsonObject != null && jsonObject.credit>0) {
row['J'] = jsonObject.get("id")
row['K'] = jsonObject.get("city")
row['L'] = jsonObject.get("district")
row['M'] = jsonObject.get("name")
def type = jsonObject.get("propertyType")
if(type && type!='null'){
row['N'] = type
}
row['O'] = jsonObject.get("credit")
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(jsonObject != null) {
row['Y'] = jsonObject.get("id")
row['Z'] = jsonObject.get("city")
row['AA'] = jsonObject.get("district")
row['AB'] = jsonObject.get("name")
}
}
})
@@ -174,19 +169,22 @@ class MsbPocTest {
def addCCBPrice(path){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['M']){
def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
if(dist){
def d = CommonDatautils.findDistrictById(dist._id.toString())
if(!row['V']){
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
// if(dist){
if(row['Y'] instanceof String){
def d = CommonDatautils.findDistrictById(row['Y'])
if(d){
def cityId = cityCodeDataMap.get(d.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
if(map){
row['M'] = map.avg_price
row['V'] = map.avg_price
// row['I'] = map.calc_type
}
}
}
// }
}
})
}
@@ -33,17 +33,31 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "C:\\Users\\gaojuan\\Desktop\\7Ô³ɽ»°¸Àýbadcase.xlsx"
// addGXDPrice(filePath)
def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx"
addGXDPrice(filePath)
// addCcbPrice(filePath)
// addThirdPrice(filePath)
// addFjwPrice(filePath)
exportPkPrice()
// exportPkPrice()
// addRegion()
// addTradePrice(filePath)
}
def addTradePrice(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['Y']){
def dist = CommonDatautils.findDistrictById(d['Y'])
def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01')
if (tradePrice) {
d['W'] = tradePrice.toInteger()
}
}
});
}
def addRegion(){
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
def dist = CommonDatautils.findDistrictById(d['A'])
@@ -153,15 +167,15 @@ class AddPriceTest {
}
def addCcbPrice(filepath){
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['M']){
def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['M'], cityId)
if(d['A']){
// def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
if(map){
d['AK'] = map.avg_price
d['AL'] = map.calc_type
d['AM'] = map.credit_value
// d['AK'] = map.avg_price
d['P'] = map.calc_type
// d['AM'] = map.credit_value
}
}
})
@@ -190,11 +204,11 @@ class AddPriceTest {
def addThirdPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['M']) {
def dist = CommonDatautils.findDistrictById(d['M'])
if (!d['X']) {
def dist = CommonDatautils.findDistrictById(d['Y'])
if(dist){
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
d['AN'] = thirdPrice
d['X'] = thirdPrice
}
}
})
@@ -206,8 +220,8 @@ class AddPriceTest {
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
q.put("s_date", 202404)
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def price = null
for(def db : dbcursor){
@@ -216,10 +230,10 @@ class AddPriceTest {
break;
}
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
price = db.beike_esf_price
break
}
// if(!price && db.beike_esf_price && db.beike_esf_price>2000){
// price = db.beike_esf_price
// break
// }
}
return price
}
@@ -227,12 +241,12 @@ class AddPriceTest {
def addGXDPrice(filepath){
ExcelUtils.multiThreadingModify(filepath, { d ->
if (d['B']) {
def dist = CommonDatautils.findDistrictById(d['E'])
if (!d['M']) {
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
d['L'] = gxdPrice
d['M'] = gxdPrice
}
}
})
@@ -256,14 +270,17 @@ class AddPriceTest {
def addFjwPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['K']){
def dist = CommonDatautils.findDistrictById(d['E'])
if(dist){
if(!d['L']){
if(d['B'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['B'])
if(dist){
// d['N'] = dist.region
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd"))
if(dbo){
// d['K'] = dbo.avg_price
d['K'] = dbo.avg_price_2b
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
// d['W'] = dbo.avg_price
// d['X'] = dbo.cal_type
d['L'] = dbo.avg_price_2b
}
}
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import groovy.json.JsonOutput
import net.sf.json.JSONObject
import okhttp3.*;
import java.io.IOException;
@@ -15,177 +16,76 @@ import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder;
class QiliangTest {
String proxyHost = 'tunnel2.qg.net';
Integer proxyPort = 14186;
String proxyUsername = '113BBC06';
String proxyPassword = '3B2FBD05B0AF';
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
// static main(args) {
// def task = new QiliangTest()
//// task.run()
// task.getHttpResult("扬州扬子江北路399号11041")
// }
static main(args) {
QiliangTest task = new QiliangTest();
// task.run()
def run(){//地址测试-2024.3.4-结果.xlsx
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
if(!row['N']){
def address = null
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
address = row['B']+row['C']
}else{
address = row['C']
}
if(address){
def result = getHttpResult(address)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['N'] = data.province
}
if(data.city && data.city!='null'){
row['O'] = data.city
}
if(data.district && data.district!='null'){
row['P'] = data.district
}
if(data.town && data.town!='null'){
row['Q'] = data.town
}
if(data.village && data.village!='null'){
row['R'] = data.village
}
if(data.areaCode && data.areaCode!='null'){
row['S'] = data.areaCode
}
if(data.farmer == "0"){
row['T'] = "非农"
}else{
row['T'] = "农"
}
if(data.isGovt == "0"){
row['U'] = "否"
}else{
row['U'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['V'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['W'] = data.govtAddr?data.govtAddr:''
}
}
}
}
}
})
def data = [
'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
print(s);
}
def getHttpResult(address){
def data = new JSONObject()
data.put("address", address)
data.put("type","2")
Map headers = new HashMap();
headers.put("Accept", "application/json, text/plain, */*");
headers.put("Accept-Encoding", "gzip, deflate");
headers.put("Accept-Language","zh-CN,zh;q=0.9")
headers.put("Content-Type","application/json;charset=UTF-8")
headers.put("Host","146.56.217.44:9010")
headers.put("Origin","http://146.56.217.44:8088")
headers.put("Proxy-Connection","keep-alive")
headers.put("Referer","http://146.56.217.44:8088/")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
HttpRequest request1 = new HttpRequest()
try{
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
def response = JSONObject.fromObject(res)
println response
return response
}catch(Exception e){
e.printStackTrace()
String getProxyResult(data){
// 将 JSON 对象转换为字符串
String jsonData = new JsonOutput().toJson(data);
// 创建代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
// 创建 URL 对象
URL url = new URL(targetUrl);
// 创建连接对象
URLConnection connection = url.openConnection(proxy);
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST");
// 允许输出流
connection.setDoOutput(true);
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json");
// 发送请求参数
OutputStream outputStream = connection.getOutputStream();
outputStream.write(jsonData.getBytes("UTF-8"));
outputStream.flush();
outputStream.close();
// 读取响应
int responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
StringBuilder response = new StringBuilder();
String line;
while ((line = reader.readLine()) != null) {
response.append(line);
}
reader.close();
return response.toString();
} else {
println "HTTP request failed with response code ${responseCode}";
}
return null;
}
final static String proxyIp = "tunnel2.qg.net";
final static Integer proxyPort = 14186;
final static String authKey = "113BBC06";
final static String password = "3B2FBD05B0AF";
public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8");
// public String request(params) throws IOException {
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort));
// OkHttpClient client = new OkHttpClient.Builder()
// .proxy(proxy)
// .proxyAuthenticator((route, response){
// String credential = Credentials.basic(authKey, password);
// return response.request().newBuilder().header("Proxy-Authorization", credential).build();
// }).
// build();
//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build();
//
// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params));
// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build();
//
// Response response = client.newCall(request).execute();
// return response.body().string();
//
//// return client.newCall(request).execute();
// }
public static void main(String[] args) {
def proxyHost = "tunnel2.qg.net"
def proxyPort = 14186
def proxyUsername = '113BBC06'
def proxyPassword = '3B2FBD05B0AF'
def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign'
def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2']
// 创建代理服务器的连接
def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}")
// 打开代理连接
def proxyConnection = proxy.openConnection()
// 创建目标URL的连接
def url = new URL(targetUrl)
def connection = url.openConnection(proxy)
// 设置请求方法
connection.setRequestMethod("GET")
// 添加请求头(可选)
connection.setRequestProperty("Content-Type", "application/json")
// 构建查询参数
def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&')
// 将查询参数附加到目标URL
def finalUrl = "${targetUrl}?${queryString}"
// 发送请求
connection.connect()
// 获取响应
def responseCode = connection.getResponseCode()
def responseMessage = connection.getResponseMessage()
// 读取响应内容
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
def responseContent = reader.readLine()
reader.close()
println "Response Code: ${responseCode}"
println "Response Message: ${responseMessage}"
println "Response Content: ${responseContent}"
}
}
@@ -18,77 +18,75 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
//def s = getTownshipInfo("118.702988,39.740979")
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
//ExcelUtils.multiThreadingModify(filePath, { row ->
//// if(!row['I']){
//// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"ÈËÃñÕþ¸®")
//// if(rets){
//// def obj = findOneByName(rets, row['C']+"ÈËÃñÕþ¸®")
//// if(obj){
//// row['I'] = obj.lng
//// row['J'] = obj.lat
//// println row['C'] + " "+obj
//// }
//// }
//// }
//
//// if(row['D'] && !row['F']){
//// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
//// def map = ["lon":row['D'], "lat":row['E']]
//// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
//// row['F'] = township
//// }
//
//// if(row['F'] && !row['J']){
//// def q = new BasicDBObject()
//// q.put("city_name", row['B'])
//// q.put("area_name", row['C'])
//// def streetName = row['F']
//// q.put("street_name", streetName)
//// def count = COMMUNITY_CONN.count(q);
//// row['J'] = count
////// row['H'] = streetName
//// }
//// if(!row['H']){
//// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
//// if(gaodeMap && gaodeMap.township){
//// row['H'] = gaodeMap.township
//// }
//// }
//
//// if(row['J'] && row['J'].toString().toDouble()>0){
//// def up_query = new BasicDBObject()
//// up_query.put("city_name", row['B'])
//// up_query.put("area_name", row['C'])
//// up_query.put("street_name", row['F'])
//// def udata = new BasicDBObject()
//// udata.put("has_gov", "ÊÇ")
//// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
//// }
//
//
//})
def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
for(def d : objs){
if(!d.name_pinyin){
if(d.name){
println d._id
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
def udata = new BasicDBObject()
udata.put("name_pinyin", py1)
if(d.location){
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
udata.put("location", ["lat":lat, "lon":lon])
ExcelUtils.modify(filePath, { row ->
if(!row['I'] && row['I']=='宝应县'){
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
if(rets){
def obj = findOneByName(rets, row['C']+"人民政府")
if(obj){
row['I'] = obj.lng
row['J'] = obj.lat
println row['C'] + " "+obj
}
def q = new BasicDBObject()
q.put("_id", d._id)
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
}
}
}
if(row['D'] && !row['F']){
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
def map = ["lon":row['D'], "lat":row['E']]
def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
row['F'] = township
}
// if(row['F'] && !row['J']){
// def q = new BasicDBObject()
// q.put("city_name", row['B'])
// q.put("area_name", row['C'])
// def streetName = row['F']
// q.put("street_name", streetName)
// def count = COMMUNITY_CONN.count(q);
// row['J'] = count
//// row['H'] = streetName
// }
// if(!row['H']){
// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
// if(gaodeMap && gaodeMap.township){
// row['H'] = gaodeMap.township
// }
// }
// if(row['J'] && row['J'].toString().toDouble()>0){
// def up_query = new BasicDBObject()
// up_query.put("city_name", row['B'])
// up_query.put("area_name", row['C'])
// up_query.put("street_name", row['F'])
// def udata = new BasicDBObject()
// udata.put("has_gov", "是")
// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
// }
})
//def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
//for(def d : objs){
// if(!d.name_pinyin){
// if(d.name){
// println d._id
// def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
// def udata = new BasicDBObject()
// udata.put("name_pinyin", py1)
// if(d.location){
// def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
// def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
// udata.put("location", ["lat":lat, "lon":lon])
// }
//
// def q = new BasicDBObject()
// q.put("_id", d._id)
// COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
// }
// }
//}
def getTownshipInfo(def glocation){
@@ -0,0 +1,181 @@
package com.fangjia.script.gj.village
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.temp.shenjian.util.HttpUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.HttpClient
import groovy.json.JsonOutput
import net.sf.json.JSONObject
import okhttp3.*;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.util.concurrent.TimeUnit;
import okhttp3.Request.Builder;
class QiliangTest1 {
def proxyHost = 'tunnel2.qg.net'
def proxyPort = 14186
def proxyUsername = '113BBC06'
def proxyPassword = '3B2FBD05B0AF'
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
static main(args) {
def task = new QiliangTest()
// task.run()
def data = [
'address': '江苏省扬州市宝应县安宜镇宝应大道88号',
'type': '1'
]
def s = task.getProxyResult(data)
print(s)
}
def run(){//地址测试-2024.3.4-结果.xlsx
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
if(!row['N']){
def address = null
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
address = row['B']+row['C']
}else{
address = row['C']
}
if(address){
def result = getHttpResult(address)
if(result && result.code==20000){
def data = result.data
if(data){
if(data.province && data.province!='null'){
row['N'] = data.province
}
if(data.city && data.city!='null'){
row['O'] = data.city
}
if(data.district && data.district!='null'){
row['P'] = data.district
}
if(data.town && data.town!='null'){
row['Q'] = data.town
}
if(data.village && data.village!='null'){
row['R'] = data.village
}
if(data.areaCode && data.areaCode!='null'){
row['S'] = data.areaCode
}
if(data.farmer == "0"){
row['T'] = "非农"
}else{
row['T'] = "农"
}
if(data.isGovt == "0"){
row['U'] = "否"
}else{
row['U'] = "是"
}
def list = data.indusParkList
if(list && list!="null"){
def s = ""
for(def r : list){
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
}
row['V'] = s
}
if(data.govtAddr && data.govtAddr!='null'){
row['W'] = data.govtAddr?data.govtAddr:''
}
}
}
}
}
})
}
JSONObject getHttpResult(address){
JSONObject data = new JSONObject()
data.put("address", address)
data.put("type","2")
Map headers = new HashMap();
headers.put("Accept", "application/json, text/plain, */*");
headers.put("Accept-Encoding", "gzip, deflate");
headers.put("Accept-Language","zh-CN,zh;q=0.9")
headers.put("Content-Type","application/json;charset=UTF-8")
headers.put("Host","146.56.217.44:9010")
headers.put("Origin","http://146.56.217.44:8088")
headers.put("Proxy-Connection","keep-alive")
headers.put("Referer","http://146.56.217.44:8088/")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
HttpRequest request1 = new HttpRequest()
try{
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
JSONObject response = JSONObject.fromObject(res)
println response
return response
}catch(Exception e){
e.printStackTrace()
}
return null;
}
def getProxyResult(data){
// 将 JSON 对象转换为字符串
def jsonData = new JsonOutput().toJson(data)
// 创建代理对象
def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))
// 创建 URL 对象
def url = new URL(targetUrl)
// 创建连接对象
def connection = url.openConnection(proxy)
// 设置代理认证
java.net.Authenticator.setDefault(new java.net.Authenticator() {
protected PasswordAuthentication getPasswordAuthentication() {
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
}
})
// 设置请求方法为 POST
connection.setRequestMethod("POST")
// 允许输出流
connection.setDoOutput(true)
// 设置请求头
connection.setRequestProperty("Content-Type", "application/json")
// 发送请求参数
def outputStream = connection.getOutputStream()
outputStream.write(jsonData.getBytes("UTF-8"))
outputStream.flush()
outputStream.close()
// 读取响应
def responseCode = connection.getResponseCode()
if (responseCode == HttpURLConnection.HTTP_OK) {
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
def response = new StringBuilder()
String line
while ((line = reader.readLine()) != null) {
response.append(line)
}
reader.close()
return response.toString()
} else {
println "HTTP request failed with response code ${responseCode}"
}
return null
}
}
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
@@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
@@ -58,6 +60,9 @@ class GaoDePoiDataProcess extends BaseBatch{
def fullCityMap = null
DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL)
static main(args) {
runBatch(new GaoDePoiDataProcess(), args)
@@ -69,12 +74,215 @@ class GaoDePoiDataProcess extends BaseBatch{
// synchMapPoi();
synchCompany()
// synchCompany()
// poiCatProcess()
// addressAndCatProcess()
// synchNetwork()
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
// sysnchPoi()
// sysnchVillage()
addTownship()
}
def addTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d->
if(d['E']){
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
// if(gaodeMap && gaodeMap.township){
// if(gaodeMap.city && gaodeMap.city!="[]"){
// d['H'] = gaodeMap.city
// }
// if( gaodeMap.district && gaodeMap.district!='[]'){
// d['I'] = gaodeMap.district
// }
// if(gaodeMap.township && gaodeMap.township!='[]'){
// d['J'] = gaodeMap.township
// }
// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
// d['K'] = gaodeMap.format_addr
// }
//
// }
def q = new BasicDBObject()
q.put("city", d['H'])
q.put("district", d['I'])
q.put("street",d['J'])
def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q)
if(obj){
d['L'] = "是"
}
}
},0)
}
def sysnchVillage(){
def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"]
for(def city : buducitys){
def pois = findAllVillagePoiMap(city)
def rows = createVillageData(city, pois)
save(city, rows)
saveToMG(city, rows)
println city +" "+pois.size()+" add:"+rows.size()
}
}
def createVillageData(city, pois){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(pois.size())
def townMap = findTownRange(city)
for(def i=0;i<pois.size();i++){
def p = pois[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
if(!obj){
def prov = ProvinceData.getProvince(p.city.replace("市",""))
def ret = createVillageData(prov, p, townMap)
rets.addAll(ret)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def isTown( city, region, name){
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
// obj = it
// },"192.168.0.89")
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
def body = """
{
"query": {
"bool":{
"must":[
{"term": {"city.keyword":"${city}"}},
{"term": {"district.keyword":"${region}"}},
{"term": {"town.keyword":"${town}"}}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
def response = JSONObject.fromObject(res)
def obj = null
if(response && response.hits){
def hits = response.hits.hits
if(hits){
obj = hits[0]
}
}
println(response)
return obj
}
def createVillageData(province, map, townMap){
// def fjw_province = province
// def fjw_city = RegionData.formatCity(map.city)
// def fjw_region = RegionData.formatRegion(fjw_city, map.district)
def retMap = [:]
retMap.province = ProvinceData.getFullProvince(province)
// retMap.fjw_province = fjw_province
// retMap.fjw_city = fjw_city
retMap.city = map.city
retMap.district = map.district
retMap.name = map.name
retMap.type = "地名地址信息"
retMap.secondary_type = "普通地名"
retMap.small_type = "村庄级地名"
retMap.cat = "地名地址信息;普通地名;村庄级地名"
retMap.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
retMap.township = InsidePolygon.isInArea2(retMap.location.lon, retMap.location.lat, townMap)
//高德转百度
def baiduMap = GeoUtils.gcj02ToBd09(retMap.location.lat, retMap.location.lon)
if(baiduMap){
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
retMap.bd_location = location
}
//高德转WGS84
def gpsLoactionMap = GeoUtils.gcjToGps84(retMap.location.lat, retMap.location.lon)
if(gpsLoactionMap){
def gpslocation = [:]
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
retMap.gps_location = gpslocation
}
if(!retMap.township){
def gaodeMap = getTownshipInfo(retMap.location.lon+","+retMap.location.lat)
if(gaodeMap && gaodeMap.township){
retMap.township = gaodeMap.township
}
// if(!fjw_region){
// if(gaodeMap && gaodeMap.district){
// fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
// }
// }
//
// if(!fjw_region){
// println fjw_city+" "+retMap.region+" 无法格式化 "+retMap.location + " "+gaodeMap
// }
// retMap.fjw_district = fjw_region
}
// retMap.fjw_district = fjw_region
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(retMap.name, " ")
retMap.version = version
retMap.data_source = "百度地图"
retMap.data_id = map._id.toString()
retMap.id = map._id.toString()
if(retMap.secondary_type && TAGS.contains(retMap.secondary_type)){
retMap.status = 0
}else{
retMap.status = 1
}
return retMap
}
def findAllVillagePoiMap(city){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
keys.put("addr", 1)
keys.put("gcj02_x", 1)
keys.put("gcj02_y", 1)
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄").append("city", city), keys).toList()
return list;
}
def sysnchPoi(){
@@ -158,7 +366,6 @@ class GaoDePoiDataProcess extends BaseBatch{
final CountDownLatch latch = new CountDownLatch(results.size())
def prov = ProvinceData.getProvince(city.replace("ÊÐ",""))
for(def i=0;i<results.size();i++){
def map = results[i]
fixedThreadPool.execute(new Runnable() {
@@ -348,7 +555,7 @@ class GaoDePoiDataProcess extends BaseBatch{
def client = ElasticUtils.getClient("192.168.0.89")
for (int i = 0; i < rows.size(); i++) {
def map = rows[i]
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
if(i%1000==0){
print "."
}
@@ -409,6 +616,8 @@ class GaoDePoiDataProcess extends BaseBatch{
def getTownshipInfo(def glocation){
def township = null
def district = null
def city = null
def format_addr = null
def jsonObject = null
def stop = false
def count = 0
@@ -420,6 +629,9 @@ class GaoDePoiDataProcess extends BaseBatch{
if(addressComponent){
township = addressComponent.township
district = addressComponent.district
city = addressComponent.city
format_addr = jsonObject.regeocode.formatted_address
}
}
}
@@ -433,8 +645,10 @@ class GaoDePoiDataProcess extends BaseBatch{
}
def map = [:]
map.city = city
map.district = district
map.township = township
map.format_addr = format_addr
return map;
}
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.xingyebank.Data
@@ -34,16 +35,18 @@ class PoiWash {
public static def regions = []
public static def nameKeys = null
def fullCityMap = CommonDatautils.queryFullNameCity()
static main(args) {
def task = new PoiWash()
// task.updateId()
// task.run()
// task.synch()
// task.check()
task.deduplicate()
// task.deduplicate()
// task.updateEsMapPoi()
// task.checkSpecialSymbols()
// task.splitRoad()
task.splitRoad()
}
def updateId(){
@@ -452,9 +455,10 @@ class PoiWash {
def splitRoad(){
//"ÈýÃ÷ÊÐ",
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
def citys = CommonDatautils.findAllCity()
for(def city : citys) {
def datas = findPoiByQuery(city, ['city','address'])
def cityObj = fullCityMap.get(city)
def datas = findPoiByQuery(cityObj.city, ['city','address'])
splitRoadRun(datas)
println city +" Íê³É"
}
@@ -472,18 +476,15 @@ class PoiWash {
void run() {
try{
totalCount.addAndGet(1)
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
if(jsonObject && jsonObject.containsKey("result")){
def dataMap = jsonObject.get("result")
if(dataMap.get("street")){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("street", dataMap.get("street"))
if(dataMap.get("streetNumber")){
udata.put("street_number", dataMap.get("streetNumber"))
}
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address)
def street = jsonObject.get("street")
def streetNumber = jsonObject.get("streetNumber")
if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){
updateTotalCount.addAndGet(1)
def udata = new BasicDBObject()
udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber"))
udata.put("m_date", new Date())
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
}
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
} catch (Exception e) {
@@ -5,6 +5,8 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
@@ -26,8 +28,8 @@ class SeaweedDataSynch extends BaseBatch{
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
@@ -40,7 +42,7 @@ class SeaweedDataSynch extends BaseBatch{
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
@@ -61,11 +63,13 @@ class SeaweedDataSynch extends BaseBatch{
run()
// println "全国district, shop, office同步完成"
//民政部编码上传
//codeRun()
// codeRun()
//println "民政部省,市,区同步完成"
//println new ObjectId("622c433afb459ab08fdab460").getDate()
// addTest()
// updateTownship()
}
def addTest(){
@@ -346,24 +350,44 @@ class SeaweedDataSynch extends BaseBatch{
def codeRun(){
// def q = new BasicDBObject()
// def dbcursor = mca_administrative_division_coll.find(q)
// def rets = []
// for(def it : dbcursor){
// def udata = new BasicDBObject()
// udata.put("_id", it._id)
// udata.put("province", it.province)
// udata.put("city", it.city)
// udata.put("district", it.district)
// udata.put("mca_province_code", it.province_code)
// udata.put("mca_city_code", it.city_code)
// udata.put("mca_district_code", it.district_code)
// udata.put("level", it.level)
// udata.put("c_date", it.c_date)
// rets << udata
//// administrative_division_100_CONN.save(udata)
//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
// }
// administrative_division_100_CONN.insert(rets)
//
def q = new BasicDBObject()
def dbcursor = mca_administrative_division_coll.find(q)
def rets = []
for(def it : dbcursor){
def udata = new BasicDBObject()
udata.put("_id", it._id)
udata.put("province", it.province)
udata.put("city", it.city)
udata.put("district", it.district)
udata.put("mca_province_code", it.province_code)
udata.put("mca_city_code", it.city_code)
udata.put("mca_district_code", it.district_code)
udata.put("level", it.level)
udata.put("c_date", it.c_date)
rets << udata
// administrative_division_100_CONN.save(udata)
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
q.put("province","澳门特别行政区")
def dbcursor = administrative_division_2022_CONN.find(q)
for(def d : dbcursor){
administrative_division_100_CONN.save(d)
}
administrative_division_100_CONN.insert(rets)
}
def updateTownship(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx", { row ->
if(row['Q'] && row['Q']=='是'){
def udata = new BasicDBObject()
udata.put("township", row['J'])
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
}
})
}
}
@@ -98,6 +98,23 @@ class InsidePolygon {
}
return list;
}
public static List<List<Point>> createPoint4(def pointList){
List<List<Point>> alllist = new ArrayList<>();
for(def pointArr : pointList){
List<Point> list = new ArrayList<>();
for(def arr : pointArr){
Point point = new Point();
double x = arr[0];
double y = arr[1];
point.setLon(x);
point.setLat(y);
list.add(point);
}
alllist << list
}
return alllist
}
/**
* 判定一个经纬度点与区域的关系
@@ -284,7 +301,7 @@ class InsidePolygon {
if(flag){
def arrs = key.split("##")
name = arrs[1];
break;
// break;
}
}
if(flag){
@@ -46,7 +46,7 @@ import java.util.regex.Pattern
* 8:華硊摯華硊梗靡脤梑
* 9:華硊摯華硊梗靡(祥樓Е郖癹秶)脤梑
* 10:⑽秞⑤饜
*11:open½Ó¿ÚÆ¥Åä
* 11:open½Ó¿ÚÆ¥Åä
* 14:⑽秞⑤饜ㄛ祥樓華硊癹秶
*/
@BatchOwner(Watcher.GaoJuan)
@@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.*
public class DbUtils{
public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com";
public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
//public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
public static final String COMMON_DB = "fangjia";
public static final int COMMON_PORT = 27017;
public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
//public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
@@ -76,7 +76,6 @@ class ElasticUtils {
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@@ -101,6 +100,30 @@ class ElasticUtils {
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
/**
* 冪帠僅蛌華硊<br>
@@ -6,6 +6,7 @@ import com.fangjia.dal.SdQuery;
import com.fangjia.dal.SortParam;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.mongo.model.AgencyDistrictTradeRank;
import com.fangjia.dal.mongo.model.AliasFang;
import com.fangjia.dal.mongo.model.DistrictFacilitys;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.service.bo.DistrictInfo;
@@ -540,10 +541,10 @@ public interface DistrictService extends ModeService<DistrictKey, DistrictBo> {
*
* @param city
* @param region
* @param masterDistrictNname
* @param masterDistrictName
* @return
*/
List<District> findSlaves(String city, String region, String masterDistrictNname);
List<District> findSlaves(String city, String region, String masterDistrictName);
/**
* 分期操作时设置此小区为人工操作标识
@@ -2717,12 +2717,12 @@ public class DistrictServiceImpl implements DistrictService {
}
@Override
public List<District> findSlaves(String city, String region, String masterDistrictNname) {
public List<District> findSlaves(String city, String region, String masterDistrictName) {
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("visible", 0));
params.add(new QueryParam("city", city));
params.add(new QueryParam("region", region));
params.add(new QueryParam("masterDistrictNname", masterDistrictNname));
params.add(new QueryParam("masterDistrictName", masterDistrictName));
return districtDao.findAllByParams(params);
}
@@ -2930,5 +2930,4 @@ public class DistrictServiceImpl implements DistrictService {
params.add(new QueryParam("$sr", new SrQuery(query)));
return districtDao.findAllByParams(params);
}
}
@@ -33,7 +33,7 @@ public interface DevelopmentZoneService {
* @param id
* @param udata
*/
public void updateDevelopmentZone(DevelopmentZone developmentZone);
public void updateDevelopmentZone(Map<String,Object> udata, String id);
/**
* 根据ID查询
@@ -51,7 +51,9 @@ public interface DevelopmentZoneService {
* @param pager
* @return
*/
public List<DevelopmentZone> findDevelopmentZone(String province,String name, Pager pager);
public List<DevelopmentZone> findDevelopmentZone(String province,String name,String level, Pager pager);
public long countDev(String province,String name,String level);
}
@@ -10,6 +10,7 @@ import org.sothis.core.util.Pager;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Pattern;
public class DevelopmentZoneServiceImpl implements DevelopmentZoneService {
private DevelopmentZoneDao developmentZoneDao;
@@ -26,8 +27,8 @@ public class DevelopmentZoneServiceImpl implements DevelopmentZoneService {
}
@Override
public void updateDevelopmentZone(DevelopmentZone developmentZone) {
developmentZoneDao.update(developmentZone);
public void updateDevelopmentZone(Map<String,Object> map,String id) {
developmentZoneDao.updateById(map,id);
}
@Override
@@ -37,19 +38,44 @@ public class DevelopmentZoneServiceImpl implements DevelopmentZoneService {
@Override
public List<DevelopmentZone> findDevelopmentZone(String province, String name, Pager pager) {
public List<DevelopmentZone> findDevelopmentZone(String province,String name,String level, Pager pager) {
List<QueryParam> params = new ArrayList<QueryParam>();
if (StringUtils.isNotBlank(province)) {
params.add(new QueryParam("province", province));
Pattern pattern = Pattern.compile(".*" + province + ".*");
params.add(new QueryParam("province", pattern));
}
if (StringUtils.isNotBlank(name)) {
params.add(new QueryParam("name", name));
Pattern pattern = Pattern.compile(".*" + name + ".*");
params.add(new QueryParam("name", pattern));
}
if (StringUtils.isNotBlank(level)) {
Pattern pattern = Pattern.compile(".*" + level + ".*");
params.add(new QueryParam("level", pattern));
}
return developmentZoneDao.findAllByParams(params, pager.getStartRow(), pager.getPageSize());
}
@Override
public long countDev(String province, String name,String level) {
List<QueryParam> params = new ArrayList<QueryParam>();
if (StringUtils.isNotBlank(province)) {
Pattern pattern = Pattern.compile(".*" + province + ".*");
params.add(new QueryParam("province", pattern));
}
if (StringUtils.isNotBlank(name)) {
Pattern pattern = Pattern.compile(".*" + name + ".*");
params.add(new QueryParam("name", pattern));
}
if (StringUtils.isNotBlank(level)) {
Pattern pattern = Pattern.compile(".*" + level + ".*");
params.add(new QueryParam("level", pattern));
}
return developmentZoneDao.findCountByParams(params);
}
public void setDevelopmentZoneDao(DevelopmentZoneDao developmentZoneDao) {
this.developmentZoneDao = developmentZoneDao;
@@ -0,0 +1,23 @@
package com.fangjia.service.open;
import com.fangjia.dal.mongo.model.AliasFang;
import java.util.List;
import java.util.Set;
/**
* @author shenjian
* @since 2024/6/11
*/
public interface AliasFangService {
/**
* 查询搜房别名与名称关系
*
* @param city
* @param region
* @param names
* @return
*/
List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names);
}
@@ -0,0 +1,27 @@
package com.fangjia.service.open;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.mongo.dao.AliasFangDao;
import com.fangjia.dal.mongo.model.AliasFang;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
/**
* @author shenjian
* @since 2024/6/11
*/
public class AliasFangServiceImpl implements AliasFangService{
private AliasFangDao aliasFangDao;
@Override
public List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names) {
return aliasFangDao.findFangByNameAndAlias(city, region, names);
}
public void setAliasFangDao(AliasFangDao aliasFangDao) {
this.aliasFangDao = aliasFangDao;
}
}