Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+125
-69
@@ -4,9 +4,13 @@ import com.fangjia.bkoff.model.DevelopmentZoneModel;
|
||||
import com.fangjia.bkoff.model.JsonModel;
|
||||
import com.fangjia.bkoff.model.LatlngPolygonModel;
|
||||
import com.fangjia.core.util.GeoUtils;
|
||||
import com.fangjia.core.util.ThreadLocalData;
|
||||
import com.fangjia.core.util.ThreadLocalKey;
|
||||
import com.fangjia.dal.mongo.model.DevelopmentZone;
|
||||
import com.fangjia.dal.mongo.model.LatlngPolygon;
|
||||
import com.fangjia.dal.mongo.model.StatsIndex;
|
||||
import com.fangjia.dal.mongo.model.bkoff.BkoffUsers;
|
||||
import com.fangjia.service.mongo.bkoff.BkoffUsersService;
|
||||
import com.fangjia.service.mongo.fangjia.DevelopmentZoneService;
|
||||
import com.fangjia.service.mongo.fangjia.LatlngPolygonService;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
@@ -22,13 +26,16 @@ import static com.fangjia.core.util.GeoUtils.LNG;
|
||||
*/
|
||||
public class DevelopmentZoneController {
|
||||
private DevelopmentZoneService developmentZoneService;
|
||||
private BkoffUsersService bkoffUsersService;
|
||||
|
||||
public Object listAction(DevelopmentZoneModel model) {
|
||||
Pager pager = model.getPager(30);
|
||||
if (pager.getCurrentPage() > 100) {
|
||||
pager.setCurrentPage(1);
|
||||
}
|
||||
List<DevelopmentZone> developmentZones = developmentZoneService.findDevelopmentZone(model.getProvince(), model.getName(), pager);
|
||||
long countDev = developmentZoneService.countDev(model.getProvince(),model.getName(), model.getLevel());
|
||||
pager.setTotalRows((int) countDev);
|
||||
List<DevelopmentZone> developmentZones = developmentZoneService.findDevelopmentZone(model.getProvince(),model.getName(), model.getLevel(), pager);
|
||||
model.setDevelopmentZoneList(developmentZones);
|
||||
model.setPager(pager);
|
||||
return model;
|
||||
@@ -62,44 +69,53 @@ public class DevelopmentZoneController {
|
||||
}
|
||||
|
||||
public Object saveOrUpdateAction(DevelopmentZoneModel model) {
|
||||
String userId = (String) ThreadLocalData.getInstance().get(ThreadLocalKey.UserId);
|
||||
BkoffUsers user = bkoffUsersService.findBkoffUsersById(userId);
|
||||
JsonModel ret = new JsonModel();
|
||||
Map<String, Object> data = new HashMap<String, Object>();
|
||||
ret.setJsonModel(data);
|
||||
// name province bdLocation level 不能为空
|
||||
if (StringUtils.isBlank(model.getProvince()) || StringUtils.isBlank(model.getName()) || model.getBdShape() == null || model.getBdShape().toString().equals("[]") || StringUtils.isBlank(model.getLevel())) {
|
||||
data.put("msg", "保存失败");
|
||||
return ret;
|
||||
}
|
||||
|
||||
if (model.getId() == null) {
|
||||
DevelopmentZone entity = new DevelopmentZone();
|
||||
// name province bdLocation level 不能为空
|
||||
if (StringUtils.isBlank(model.getProvince()) || StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getLevel())) {
|
||||
data.put("msg", "保存失败(开发区名称,所在省份,级别不能为空)");
|
||||
return ret;
|
||||
}
|
||||
DevelopmentZone entity = new DevelopmentZone();
|
||||
// name province bdLocation level 不能为空/
|
||||
entity.setName(model.getName());
|
||||
entity.setProvince(model.getProvince());
|
||||
String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", "");
|
||||
List<List<Double>> bdLocationList = new ArrayList<List<Double>>();
|
||||
String[] splitStr = locationStr.split(";");
|
||||
for (int i = 0; i < splitStr.length; i++) {
|
||||
String[] splitStr2 = splitStr[i].split(",");
|
||||
List<Double> location = new ArrayList<Double>();
|
||||
location.add(Double.parseDouble(splitStr2[0]));
|
||||
location.add(Double.parseDouble(splitStr2[1]));
|
||||
bdLocationList.add(location);
|
||||
// 新加开发区,围栏不是必须有的
|
||||
if (model.getBdShape() != null && !model.getBdShape().toString().equals("[]")) {
|
||||
String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", "");
|
||||
List<List<Double>> bdLocationList = new ArrayList<List<Double>>();
|
||||
String[] splitStr = locationStr.split(";");
|
||||
for (int i = 0; i < splitStr.length; i++) {
|
||||
String[] splitStr2 = splitStr[i].split(",");
|
||||
List<Double> location = new ArrayList<Double>();
|
||||
location.add(Double.parseDouble(splitStr2[0]));
|
||||
location.add(Double.parseDouble(splitStr2[1]));
|
||||
bdLocationList.add(location);
|
||||
}
|
||||
//加上第一个,形成首尾闭合
|
||||
bdLocationList.add(bdLocationList.get(0));
|
||||
model.setBdShape(bdLocationList);
|
||||
// 百度经纬度
|
||||
Map<String, Object> bdShape = new HashMap<String, Object>();
|
||||
bdShape.put("type", "polygon");
|
||||
bdShape.put("coordinates", model.getBdShape());
|
||||
entity.setBdShape(bdShape);
|
||||
// 高德经纬度
|
||||
Map<String, Object> shape = new HashMap<String, Object>();
|
||||
shape.put("type", "polygon");
|
||||
Object coordinates = convertGeoList(model.getBdShape());
|
||||
shape.put("coordinates", coordinates);
|
||||
entity.setShape(shape);
|
||||
}
|
||||
model.setBdShape(bdLocationList);
|
||||
// 百度经纬度
|
||||
Map<String, Object> bdShape = new HashMap<String, Object>();
|
||||
bdShape.put("type", "polygon");
|
||||
bdShape.put("coordinates", model.getBdShape());
|
||||
entity.setBdShape(bdShape);
|
||||
// 高德经纬度
|
||||
Map<String, Object> shape = new HashMap<String, Object>();
|
||||
shape.put("type", "polygon");
|
||||
Object coordinates = convertGeoList(model.getBdShape());
|
||||
shape.put("coordinates", coordinates);
|
||||
entity.setShape(shape);
|
||||
|
||||
entity.setLevel(model.getLevel());
|
||||
String aliasString = model.getAlias().replaceAll(",",",");
|
||||
List<String> alias = Arrays.asList(aliasString.split("\\s*,\\s*"));
|
||||
entity.setAlias(alias);
|
||||
entity.setAddress(StringUtils.isBlank(model.getAddress()) ? "" : model.getAddress());
|
||||
entity.setArea(StringUtils.isBlank(model.getArea()) ? "" : model.getArea());
|
||||
entity.setBlock(StringUtils.isBlank(model.getBlock()) ? "" : model.getBlock());
|
||||
@@ -109,50 +125,86 @@ public class DevelopmentZoneController {
|
||||
entity.setNameEn(StringUtils.isBlank(model.getNameEn()) ? "" : model.getNameEn());
|
||||
entity.setType(StringUtils.isBlank(model.getType()) ? "" : model.getType());
|
||||
entity.setUrl(StringUtils.isBlank(model.getUrl()) ? "" : model.getUrl());
|
||||
entity.setCityName(StringUtils.isBlank(model.getCityName()) ? "" : model.getCityName());
|
||||
//记录操作人员
|
||||
entity.setOperator(user.getRealName());
|
||||
developmentZoneService.saveDevelopmentZone(entity);
|
||||
data.put("msg", "保存成功");
|
||||
} else {
|
||||
DevelopmentZone entity = developmentZoneService.findDevelopmentZone(model.getId());
|
||||
// Map<String, Object> udata = new HashMap<String, Object>();
|
||||
|
||||
// name province bdLocation level 不能为空
|
||||
entity.setName(model.getName());
|
||||
entity.setProvince(model.getProvince());
|
||||
String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", "");
|
||||
List<List<Double>> bdLocationList = new ArrayList<List<Double>>();
|
||||
String[] splitStr = locationStr.split(";");
|
||||
for (int i = 0; i < splitStr.length; i++) {
|
||||
String[] splitStr2 = splitStr[i].split(",");
|
||||
List<Double> location = new ArrayList<Double>();
|
||||
location.add(Double.parseDouble(splitStr2[0]));
|
||||
location.add(Double.parseDouble(splitStr2[1]));
|
||||
bdLocationList.add(location);
|
||||
// DevelopmentZone entity = developmentZoneService.findDevelopmentZone(model.getId());
|
||||
Map<String, Object> udata = new HashMap<String, Object>();
|
||||
if (StringUtils.isNotBlank(model.getProvince())) {
|
||||
udata.put("province", model.getProvince());
|
||||
}
|
||||
model.setBdShape(bdLocationList);
|
||||
// 百度经纬度
|
||||
Map<String, Object> bdShape = new HashMap<String, Object>();
|
||||
bdShape.put("type", "polygon");
|
||||
bdShape.put("coordinates", model.getBdShape());
|
||||
entity.setBdShape(bdShape);
|
||||
// 高德经纬度
|
||||
Map<String, Object> shape = new HashMap<String, Object>();
|
||||
shape.put("type", "polygon");
|
||||
Object coordinates = convertGeoList(model.getBdShape());
|
||||
shape.put("coordinates", coordinates);
|
||||
entity.setShape(shape);
|
||||
|
||||
entity.setLevel(model.getLevel());
|
||||
entity.setAddress(StringUtils.isBlank(model.getAddress()) ? "" : model.getAddress());
|
||||
entity.setArea(StringUtils.isBlank(model.getArea()) ? "" : model.getArea());
|
||||
entity.setBlock(StringUtils.isBlank(model.getBlock()) ? "" : model.getBlock());
|
||||
entity.setBuildDate(StringUtils.isBlank(model.getBuildDate()) ? "" : model.getBuildDate());
|
||||
entity.setCreateTime(new Date());
|
||||
entity.setIndustry(StringUtils.isBlank(model.getIndustry()) ? "" : model.getIndustry());
|
||||
entity.setNameEn(StringUtils.isBlank(model.getNameEn()) ? "" : model.getNameEn());
|
||||
entity.setType(StringUtils.isBlank(model.getType()) ? "" : model.getType());
|
||||
entity.setUrl(StringUtils.isBlank(model.getUrl()) ? "" : model.getUrl());
|
||||
developmentZoneService.updateDevelopmentZone(entity);
|
||||
data.put("msg", "保存成功");
|
||||
if (StringUtils.isNotBlank(model.getName())) {
|
||||
udata.put("name", model.getName());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getLevel())) {
|
||||
udata.put("level", model.getLevel());
|
||||
}
|
||||
if (model.getBdShape() != null && !model.getBdShape().toString().equals("[]")) {
|
||||
String locationStr = model.getBdShape().toString().replaceAll("\\[", "").replaceAll("\\]", "").replaceAll(" ", "");
|
||||
List<List<Double>> bdLocationList = new ArrayList<List<Double>>();
|
||||
String[] splitStr = locationStr.split(";");
|
||||
for (int i = 0; i < splitStr.length; i++) {
|
||||
String[] splitStr2 = splitStr[i].split(",");
|
||||
List<Double> location = new ArrayList<Double>();
|
||||
location.add(Double.parseDouble(splitStr2[0]));
|
||||
location.add(Double.parseDouble(splitStr2[1]));
|
||||
bdLocationList.add(location);
|
||||
}
|
||||
//加上第一个,形成首尾闭合
|
||||
bdLocationList.add(bdLocationList.get(0));
|
||||
model.setBdShape(bdLocationList);
|
||||
// 百度经纬度
|
||||
Map<String, Object> bdShape = new HashMap<String, Object>();
|
||||
bdShape.put("type", "polygon");
|
||||
bdShape.put("coordinates", model.getBdShape());
|
||||
udata.put("bdShape", bdShape);
|
||||
// 高德经纬度
|
||||
Map<String, Object> shape = new HashMap<String, Object>();
|
||||
shape.put("type", "polygon");
|
||||
Object coordinates = convertGeoList(model.getBdShape());
|
||||
shape.put("coordinates", coordinates);
|
||||
udata.put("shape", shape);
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getAlias()) ){
|
||||
String aliasString = model.getAlias().replaceAll(",",",");
|
||||
List<String> alias = Arrays.asList(aliasString.split("\\s*,\\s*"));
|
||||
udata.put("alias", alias);
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getAddress())) {
|
||||
udata.put("address", model.getAddress());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getArea())) {
|
||||
udata.put("area", model.getArea());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getBlock())) {
|
||||
udata.put("block", model.getBlock());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getBuildDate())) {
|
||||
udata.put("buildDate", model.getBuildDate());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getIndustry())) {
|
||||
udata.put("industry", model.getIndustry());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getNameEn())) {
|
||||
udata.put("nameEn", model.getNameEn());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getType())) {
|
||||
udata.put("type", model.getType());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getUrl())) {
|
||||
udata.put("url", model.getUrl());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getCityName())) {
|
||||
udata.put("cityName", model.getCityName());
|
||||
}
|
||||
udata.put("createTime", new Date());
|
||||
//记录操作人员
|
||||
udata.put("operator", user.getRealName());
|
||||
developmentZoneService.updateDevelopmentZone(udata, model.getId());
|
||||
data.put("msg", "修改成功");
|
||||
}
|
||||
|
||||
return ret;
|
||||
@@ -162,6 +214,10 @@ public class DevelopmentZoneController {
|
||||
public void setDevelopmentZoneService(DevelopmentZoneService developmentZoneService) {
|
||||
this.developmentZoneService = developmentZoneService;
|
||||
}
|
||||
|
||||
public void setBkoffUsersService(BkoffUsersService bkoffUsersService) {
|
||||
this.bkoffUsersService = bkoffUsersService;
|
||||
}
|
||||
//将百度坐标转为高德坐标
|
||||
|
||||
public List<List<Double>> convertGeoList(List<List<Double>> bdLocationList) {
|
||||
|
||||
+19
-1
@@ -17,20 +17,38 @@ public class DevelopmentZoneModel extends BaseModel {
|
||||
|
||||
private String level;
|
||||
private String name;
|
||||
private String alias;
|
||||
// private List<String> addr_alias;
|
||||
private String nameEn;
|
||||
private String province;
|
||||
private String type;
|
||||
private String url;
|
||||
private String district;
|
||||
|
||||
List<List<Double>> shape;
|
||||
List<List<Double>> bdShape;
|
||||
|
||||
|
||||
private String block;
|
||||
|
||||
private DevelopmentZone developmentZone;
|
||||
private List<DevelopmentZone> developmentZoneList;
|
||||
|
||||
public String getAlias() {
|
||||
return alias;
|
||||
}
|
||||
|
||||
public void setAlias(String alias) {
|
||||
this.alias = alias;
|
||||
}
|
||||
|
||||
public String getDistrict() {
|
||||
return district;
|
||||
}
|
||||
|
||||
public void setDistrict(String district) {
|
||||
this.district = district;
|
||||
}
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
<div class="city-border">
|
||||
<div class="city-border-left hide" id="city-border-left"></div>
|
||||
<div id="city-border-mid" class="">
|
||||
<a title="选择城市" class="city-current" id="wt-city-title"><#if defaultValue?? && defaultValue != ''>${cityDisplayMap[defaultValue]}<#elseif showClear>无<#elseif showAll>全部<#else>选择城市</#if></a>
|
||||
<a title="选择城市" class="city-current" id="wt-city-title"><#if defaultValue?? && defaultValue != '' &&cityDisplayMap[defaultValue]??>${cityDisplayMap[defaultValue]}<#elseif showClear>无<#elseif showAll>全部<#else>选择城市</#if></a>
|
||||
<span class="downtip" id="wt-city-other"></span>
|
||||
</div>
|
||||
<div class="city-border-right hide" id="city-border-right"></div>
|
||||
|
||||
@@ -3,147 +3,216 @@
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
||||
<html xmlns="http://www.w3.org/1999/xhtml" xmlns="http://www.w3.org/1999/html">
|
||||
<head>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<title>开发区围栏列表</title>
|
||||
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
|
||||
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
||||
<title>开发区围栏列表</title>
|
||||
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
|
||||
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<#include "/common_js.ftl" />
|
||||
<#include "/layout/header.ftl" />
|
||||
<div class="main-content">
|
||||
<div class="menuButton">
|
||||
<a href="../developmentZone/list" class="list">开发区列表</a>
|
||||
</div>
|
||||
<#include "/common_js.ftl" />
|
||||
<#include "/layout/header.ftl" />
|
||||
<div class="main-content">
|
||||
<div class="menuButton">
|
||||
<a href="../developmentZone/list" class="list">开发区列表</a>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<form id="form1">
|
||||
<#if developmentZone??>
|
||||
<input id="id" name="id" type="hidden" value="${developmentZone.id!}"/>
|
||||
</#if>
|
||||
<div>
|
||||
<form id="form1">
|
||||
<#if developmentZone??>
|
||||
<input id="id" name="id" type="hidden" value="${developmentZone.id!}"/>
|
||||
</#if>
|
||||
|
||||
<table>
|
||||
<tr>
|
||||
<td>
|
||||
<#if latlngPolygon??>
|
||||
<@d_city defaultValue="${latlngPolygon.city!}" callback="changeCity" callbackField="display" name="cityDisplay" />
|
||||
<#else>
|
||||
<@d_city defaultValue="上海" callback="changeCity" callbackField="display" name="cityDisplay" />
|
||||
</#if>
|
||||
</td>
|
||||
<td>
|
||||
<label for="name">开发区名称:</label> <input id="name" name="name" type="text" value="<#if developmentZone??>${developmentZone.name!}</#if>"/>
|
||||
<label for="nameEn">开发区名称拼音:</label> <input id="nameEn" name="nameEn" type="text" value="<#if developmentZone??>${developmentZone.nameEn!}</#if>"/>
|
||||
<label for="province">所在省份:</label> <input id="province" name="province" type="text" value="<#if developmentZone??>${developmentZone.province!}</#if>"/>
|
||||
<label for="level">级别:</label> <input id="level" name="level" type="text" value="<#if developmentZone??>${developmentZone.level!}</#if>"/>
|
||||
<label for="address">地址:</label> <input id="address" name="address" type="text" value="<#if developmentZone??>${developmentZone.address!}</#if>"/>
|
||||
<label for="area">所在地区:</label> <input id="area" name="area" type="text" value="<#if developmentZone??>${developmentZone.area!}</#if>"/>
|
||||
<label for="buildDate">建立年代:</label><input id="buildDate" name="buildDate" type="text" value="<#if developmentZone??>${developmentZone.buildDate!}</#if>"/>
|
||||
<label for="type">类型:</label><input id="type" name="type" type="text" value="<#if developmentZone??>${developmentZone.type!}</#if>"/>
|
||||
<label for="url">URL:</label><input id="url" name="url" type="text" value="<#if developmentZone??>${developmentZone.url!}</#if>"/>
|
||||
<label for="industry">主导产业:</label><input id="industry" name="industry" type="text" value="<#if developmentZone??>${developmentZone.industry!}</#if>"/>
|
||||
<label for="block">区块:</label><input id="block" name="block" type="text" value="<#if developmentZone??>${developmentZone.block!}</#if>"/>
|
||||
<input id="saveOrUpdate" type="button" value="提交"/>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td colspan="2">
|
||||
<input name="bdShape" id="bdShape" type="hidden"/>
|
||||
<div id="allmap" style="width:100%;height:900px;"></div>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
<table>
|
||||
<tr>
|
||||
<td>
|
||||
<#if developmentZone??>
|
||||
<@d_city defaultValue="${developmentZone.cityName!}" callback="changeCity" callbackField="display" name="cityDisplay" />
|
||||
<#else>
|
||||
<@d_city defaultValue="上海" callback="changeCity" callbackField="display" name="cityDisplay" />
|
||||
</#if>
|
||||
</td>
|
||||
<td>
|
||||
<label for="name">开发区名称:</label> <input id="name" name="name" type="text" placeholder="必填" value="<#if developmentZone??>${developmentZone.name!}</#if>"/>
|
||||
<label for="alias">别名:</label>
|
||||
<input id="alias" name="alias" type="text" placeholder="多个别名用,隔开" value="<#if developmentZone??><#list (developmentZone.alias)! as alia>${alia + ','}</#list><#else></#if>"/>
|
||||
<label for="nameEn">开发区名称拼音:</label> <input id="nameEn" name="nameEn" type="text" value="<#if developmentZone??>${developmentZone.nameEn!}</#if>"/>
|
||||
<label for="province">所在省份:</label> <input id="province" name="province" type="text" placeholder="必填" value="<#if developmentZone??>${developmentZone.province!}</#if>"/><br>
|
||||
<label for="cityName">城市:</label><input id="cityName" name="cityName" type="text" value="<#if developmentZone??>${developmentZone.cityName!}</#if>"/>
|
||||
<label for="district">区域:</label><input id="district" name="district" type="text" value="<#if developmentZone??>${developmentZone.district!}</#if>"/>
|
||||
<label for="level">级别:</label> <input id="level" name="level" type="text" placeholder="必填" value="<#if developmentZone??>${developmentZone.level!}</#if>"/>
|
||||
<label for="address">地址:</label> <input id="address" name="address" type="text" value="<#if developmentZone??>${developmentZone.address!}</#if>"/><br>
|
||||
<label for="area">所在地区:</label> <input id="area" name="area" type="text" value="<#if developmentZone??>${developmentZone.area!}</#if>"/>
|
||||
<label for="buildDate">建立年代:</label><input id="buildDate" name="buildDate" type="text" value="<#if developmentZone??>${developmentZone.buildDate!}</#if>"/>
|
||||
<label for="type">类型:</label><input id="type" name="type" type="text" value="<#if developmentZone??>${developmentZone.type!}</#if>"/>
|
||||
<label for="url">URL:</label><input id="url" name="url" type="text" value="<#if developmentZone??>${developmentZone.url!}</#if>"/><br>
|
||||
<label for="industry">主导产业:</label><input id="industry" name="industry" type="text" value="<#if developmentZone??>${developmentZone.industry!}</#if>"/>
|
||||
<label for="block">区块:</label><input id="block" name="block" type="text" value="<#if developmentZone??>${developmentZone.block!}</#if>"/><br/>
|
||||
<input id="saveOrUpdate" type="button" value="提交"/>
|
||||
</td>
|
||||
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<!--加载鼠标绘制工具-->
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"></script>
|
||||
<link rel="stylesheet" href="http://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.css" />
|
||||
<!--加载检索信息窗口-->
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"></script>
|
||||
<link rel="stylesheet" href="http://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.css" />
|
||||
</tr>
|
||||
<tr>
|
||||
<td colspan="2">
|
||||
<input name="bdShape" id="bdShape" type="hidden"/>
|
||||
<div id="allmap" style="width:100%;height:900px;"></div>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script type="application/javascript">
|
||||
var map = new BMap.Map("allmap"); // 创建Map实例
|
||||
if($("#cityDisplay").val()) {
|
||||
map.centerAndZoom($("#cityDisplay").val(), 13);
|
||||
}else {
|
||||
map.centerAndZoom('上海', 13);
|
||||
}
|
||||
map.enableScrollWheelZoom(true);
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<!--加载鼠标绘制工具-->
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"></script>
|
||||
<link rel="stylesheet" href="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.css" />
|
||||
<!--加载检索信息窗口-->
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"></script>
|
||||
<link rel="stylesheet" href="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.css" />
|
||||
|
||||
function changeCity(display){
|
||||
clearAll();
|
||||
map.centerAndZoom(display, 13);
|
||||
}
|
||||
<script type="application/javascript">
|
||||
var map = new BMap.Map("allmap"); // 创建Map实例
|
||||
if($("#cityDisplay").val()) {
|
||||
map.centerAndZoom($("#cityDisplay").val(), 13);
|
||||
}else {
|
||||
map.centerAndZoom('上海', 13);
|
||||
}
|
||||
map.enableScrollWheelZoom(true);
|
||||
|
||||
var overlays = [];
|
||||
var overlaycomplete = function(e){
|
||||
overlays.push(e.overlay);
|
||||
var points = e.overlay.getPath();
|
||||
function changeCity(display){
|
||||
clearAll();
|
||||
map.centerAndZoom(display, 13);
|
||||
}
|
||||
|
||||
var bdShape = [];
|
||||
for(var i=0;i<points.length;i++) {
|
||||
var location = [];
|
||||
location.push(points[i].lng );
|
||||
location.push(points[i].lat );
|
||||
bdShape.push(location);
|
||||
}
|
||||
$("#bdShape").val(bdShape.join(";"));
|
||||
};
|
||||
var overlays = [];
|
||||
var overlaycomplete = function(e){
|
||||
overlays.push(e.overlay);
|
||||
var points = e.overlay.getPath();
|
||||
|
||||
var styleOptions = {
|
||||
strokeColor:"red", //边线颜色。
|
||||
fillColor:"red", //填充颜色。当参数为空时,圆形将没有填充效果。
|
||||
strokeWeight: 3, //边线的宽度,以像素为单位。
|
||||
strokeOpacity: 0.8, //边线透明度,取值范围0 - 1。
|
||||
fillOpacity: 0.6, //填充的透明度,取值范围0 - 1。
|
||||
strokeStyle: 'solid' //边线的样式,solid或dashed。
|
||||
var bdShape = [];
|
||||
for(var i=0;i<points.length;i++) {
|
||||
var location = [];
|
||||
location.push(points[i].lng );
|
||||
location.push(points[i].lat );
|
||||
bdShape.push(location);
|
||||
}
|
||||
//实例化鼠标绘制工具
|
||||
var drawingManager = new BMapLib.DrawingManager(map, {
|
||||
isOpen: false, //是否开启绘制模式
|
||||
enableDrawingTool: true, //是否显示工具栏
|
||||
drawingToolOptions: {
|
||||
anchor: BMAP_ANCHOR_TOP_RIGHT, //位置
|
||||
offset: new BMap.Size(5, 5), //偏离值
|
||||
},
|
||||
circleOptions: styleOptions, //圆的样式
|
||||
polylineOptions: styleOptions, //线的样式
|
||||
polygonOptions: styleOptions, //多边形的样式
|
||||
rectangleOptions: styleOptions //矩形的样式
|
||||
});
|
||||
//添加鼠标绘制工具监听事件,用于获取绘制结果
|
||||
drawingManager.addEventListener('overlaycomplete', overlaycomplete);
|
||||
function clearAll() {
|
||||
for(var i = 0; i < overlays.length; i++){
|
||||
map.removeOverlay(overlays[i]);
|
||||
$("#bdShape").val(bdShape.join(";"));
|
||||
// 在创建多边形后立即启用编辑
|
||||
e.overlay.enableEditing();
|
||||
};
|
||||
|
||||
var styleOptions = {
|
||||
strokeColor:"red", //边线颜色。
|
||||
fillColor:"red", //填充颜色。当参数为空时,圆形将没有填充效果。
|
||||
strokeWeight: 1, //边线的宽度,以像素为单位。
|
||||
strokeOpacity: 0.8, //边线透明度,取值范围0 - 1。
|
||||
fillOpacity: 0.6, //填充的透明度,取值范围0 - 1。
|
||||
strokeStyle: 'solid' //边线的样式,solid或dashed。
|
||||
}
|
||||
//实例化鼠标绘制工具
|
||||
var drawingManager = new BMapLib.DrawingManager(map, {
|
||||
isOpen: true, //是否开启绘制模式
|
||||
enableDrawingTool: true, //是否显示工具栏
|
||||
drawingToolOptions: {
|
||||
anchor: BMAP_ANCHOR_TOP_RIGHT, //位置
|
||||
offset: new BMap.Size(5, 5), //偏离值
|
||||
},
|
||||
circleOptions: styleOptions, //圆的样式
|
||||
polylineOptions: styleOptions, //线的样式
|
||||
polygonOptions: styleOptions, //多边形的样式
|
||||
rectangleOptions: styleOptions //矩形的样式
|
||||
});
|
||||
//添加鼠标绘制工具监听事件,用于获取绘制结果
|
||||
drawingManager.addEventListener('overlaycomplete', overlaycomplete);
|
||||
function clearAll() {
|
||||
for(var i = 0; i < overlays.length; i++){
|
||||
map.removeOverlay(overlays[i]);
|
||||
}
|
||||
overlays.length = 0;
|
||||
}
|
||||
|
||||
// 添加点击覆盖物事件监听器
|
||||
map.addEventListener("click", function(e) {
|
||||
// 判断点击的是否是已存在的覆盖物
|
||||
if (e.overlay) {
|
||||
// 如果是多边形或折线,则可以编辑
|
||||
if (e.overlay instanceof BMap.Polygon || e.overlay instanceof BMap.Polyline) {
|
||||
// 开启覆盖物编辑功能
|
||||
e.overlay.enableEditing();
|
||||
}
|
||||
// 如果是标记点,则可以拖动
|
||||
else if (e.overlay instanceof BMap.Marker) {
|
||||
// 开启标记点拖动功能
|
||||
e.overlay.enableDragging();
|
||||
}
|
||||
overlays.length = 0;
|
||||
}
|
||||
});
|
||||
|
||||
$("#saveOrUpdate").click(function(){
|
||||
jQuery.ajax({
|
||||
type: 'POST',
|
||||
url: '../developmentZone/saveOrUpdate',
|
||||
data: $("#form1").serialize(),
|
||||
dataType: 'json',
|
||||
timeout: 100000,
|
||||
success: function(data) {
|
||||
alert(data.msg);
|
||||
location.href="list";
|
||||
},
|
||||
error : function() {
|
||||
// 添加覆盖物编辑完成事件监听器
|
||||
map.addEventListener("overlaycomplete", function(e) {
|
||||
// 判断新增的覆盖物是否是多边形或折线
|
||||
if (e.overlay instanceof BMap.Polygon || e.overlay instanceof BMap.Polyline) {
|
||||
// 添加覆盖物编辑完成事件监听器
|
||||
e.overlay.addEventListener("lineupdate", function() {
|
||||
// 编辑完成后获取更新后的路径点,并更新表单中的数据
|
||||
var points = e.overlay.getPath();
|
||||
var bdShape = [];
|
||||
for (var i = 0; i < points.length; i++) {
|
||||
var location = [];
|
||||
location.push(points[i].lng);
|
||||
location.push(points[i].lat);
|
||||
bdShape.push(location);
|
||||
}
|
||||
$("#bdShape").val(bdShape.join(";"));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
})
|
||||
$("#saveOrUpdate").click(function(){
|
||||
// 更新表单中的点位置数据
|
||||
updatePolygonPoints();
|
||||
|
||||
jQuery.ajax({
|
||||
type: 'POST',
|
||||
url: '../developmentZone/saveOrUpdate',
|
||||
data: $("#form1").serialize(),
|
||||
dataType: 'json',
|
||||
timeout: 100000,
|
||||
success: function(data) {
|
||||
alert(data.msg);
|
||||
location.href="list";
|
||||
},
|
||||
error : function() {
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
// 更新表单中的多边形点位置数据
|
||||
function updatePolygonPoints() {
|
||||
var polygons = map.getOverlays(BMap.Polygon);
|
||||
for (var i = 0; i < polygons.length; i++) {
|
||||
// 确保当前对象是一个多边形对象
|
||||
if (polygons[i] instanceof BMap.Polygon) {
|
||||
var points = polygons[i].getPath();
|
||||
var bdShape = [];
|
||||
for (var j = 0; j < points.length; j++) {
|
||||
var location = [];
|
||||
location.push(points[j].lng);
|
||||
location.push(points[j].lat);
|
||||
bdShape.push(location);
|
||||
}
|
||||
// 将更新后的点位置数据更新到表单中
|
||||
$("#bdShape").val(bdShape.join(";"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
</#compress>
|
||||
</#compress>
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
<#include "/layout/header.ftl" />
|
||||
<div class="main-content">
|
||||
<div class="menuButton">
|
||||
<a href="../developmentZone/edit" target="_blank" class="list">新增开发区围栏</a>
|
||||
<a href="../developmentZone/edit" target="_blank" class="list">新增开发区</a>
|
||||
</div>
|
||||
<form action="../developmentZone/list" id="_search_log_form">
|
||||
<div>
|
||||
@@ -24,32 +24,58 @@
|
||||
<label for="name">名称:</label>
|
||||
<input type="text" name="name" id="name" class="input-medium" style="margin-top:9px;"
|
||||
value="${name!}"></input>
|
||||
<label for="name">开发区级别:</label>
|
||||
<input type="text" name="level" id="level" class="input-medium" style="margin-top:9px;"
|
||||
value="${level!}"></input>
|
||||
|
||||
<input id="queryBtn" class="btn btn-primary" type="submit" value="查询"/>
|
||||
|
||||
</div>
|
||||
</form>
|
||||
<div class="list" style="margin-top:5px;">
|
||||
<span style="font-size:12px;">查询到<b style="color:red;">${pager.totalRows!}</b>条记录</span>
|
||||
<table style="float:left;" name="reginTable" class="regionTab">
|
||||
<tr>
|
||||
<th>名称</th>
|
||||
<th>级别</th>
|
||||
<th>别名</th>
|
||||
<th>开发区级别</th>
|
||||
<th>省份</th>
|
||||
<th style="width: 160px">建立时间</th>
|
||||
<th>城市</th>
|
||||
<th>区域</th>
|
||||
<th>区块</th>
|
||||
<th>是否有围栏</th>
|
||||
<th>操作人员</th>
|
||||
<th>修改日期</th>
|
||||
<th style="width: 120px;">操作</th>
|
||||
</tr>
|
||||
<#list developmentZoneList as it>
|
||||
<tr>
|
||||
<td>${it.name!}</td>
|
||||
<td>
|
||||
<#if it.alias??>
|
||||
<#list it.alias as item>
|
||||
${item?string}<#if item_has_next>, </#if>
|
||||
</#list>
|
||||
</#if>
|
||||
</td>
|
||||
|
||||
|
||||
<td>${it.level!}</td>
|
||||
<td>${it.province!}</td>
|
||||
<td>${it.cityName!}</td>
|
||||
<td>${it.district!}</td>
|
||||
<td>${it.block!}</td>
|
||||
<td>
|
||||
${it.buildDate}
|
||||
<#if it.bdShape?has_content>
|
||||
已有围栏
|
||||
</#if>
|
||||
</td>
|
||||
<td>${it.operator!}</td>
|
||||
<td>${it.createTime!?string("yyyy-MM-dd HH:mm:ss")}</td>
|
||||
<td>
|
||||
<a href="show?id=${it.id!}" target="_blank">查看</a>
|
||||
<a href="../developmentZone/edit?id=${it.id!}" target="_blank">新建围栏</a>
|
||||
<a name="del${it_index}" href="javascript:;" ref="delete?id=${it.id!}">删除</a>
|
||||
<#-- <a name="del${it_index}" href="javascript:;" ref="delete?id=${it.id!}">删除</a>-->
|
||||
</td>
|
||||
|
||||
</tr>
|
||||
|
||||
@@ -31,6 +31,14 @@
|
||||
<td>省份</td>
|
||||
<td>${developmentZone.province!}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>城市</td>
|
||||
<td>${developmentZone.cityName!}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>区域</td>
|
||||
<td>${developmentZone.district!}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>级别</td>
|
||||
<td>${developmentZone.level!}</td>
|
||||
@@ -39,6 +47,16 @@
|
||||
<td>名称</td>
|
||||
<td>${developmentZone.name!}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>别名</td>
|
||||
<td>
|
||||
<#if developmentZone.alias??>
|
||||
<#list developmentZone.alias as item>
|
||||
${item?string}<#if item_has_next>, </#if>
|
||||
</#list>
|
||||
</#if>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>区块</td>
|
||||
<td>${developmentZone.block!}</td>
|
||||
@@ -48,21 +66,22 @@
|
||||
<div id="allmap" style="width:1024px;height:600px;"></div>
|
||||
</td>
|
||||
</tr>
|
||||
<input id="saveOrUpdate" type="button" value="提交"/>
|
||||
</table>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script type="text/javascript"
|
||||
src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<!--加载鼠标绘制工具-->
|
||||
<script type="text/javascript"
|
||||
src="http://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"></script>
|
||||
<link rel="stylesheet" href="http://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.css"/>
|
||||
src="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.js"></script>
|
||||
<link rel="stylesheet" href="https://api.map.baidu.com/library/DrawingManager/1.4/src/DrawingManager_min.css"/>
|
||||
<!--加载检索信息窗口-->
|
||||
<script type="text/javascript"
|
||||
src="http://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"></script>
|
||||
<link rel="stylesheet" href="http://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.css"/>
|
||||
src="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.js"></script>
|
||||
<link rel="stylesheet" href="https://api.map.baidu.com/library/SearchInfoWindow/1.4/src/SearchInfoWindow_min.css"/>
|
||||
|
||||
<script type="application/javascript">
|
||||
var map = new BMap.Map("allmap"); // 创建Map实例
|
||||
@@ -71,7 +90,7 @@
|
||||
<#if developmentZone.bdShape??>
|
||||
var polygon = new BMap.Polygon([
|
||||
<#list developmentZone.bdShape.coordinates as latlng>
|
||||
new BMap.Point(${latlng[0]}, ${latlng[1]})<#if latlng_index lt developmentZone.bdShape.coordinates?size>, </#if>
|
||||
new BMap.Point(${latlng[0]?string("#.######")}, ${latlng[1]?string("#.######")})<#if latlng_index lt developmentZone.bdShape.coordinates?size>, </#if>
|
||||
</#list>
|
||||
], {strokeColor: "blue", strokeWeight: 2, strokeOpacity: 0.5});
|
||||
</#if>
|
||||
@@ -79,6 +98,48 @@
|
||||
|
||||
map.centerAndZoom(polygon.getBounds().getCenter(), 13);
|
||||
map.addOverlay(polygon); //增加多边形
|
||||
// 添加可编辑功能
|
||||
polygon.enableEditing();
|
||||
|
||||
$("#saveOrUpdate").click(function(){
|
||||
// 更新表单中的点位置数据
|
||||
updatePolygonPoints();
|
||||
|
||||
jQuery.ajax({
|
||||
type: 'POST',
|
||||
url: '../developmentZone/saveOrUpdate',
|
||||
data: $("#form1").serialize(),
|
||||
dataType: 'json',
|
||||
timeout: 100000,
|
||||
success: function(data) {
|
||||
alert(data.msg);
|
||||
location.href="list";
|
||||
},
|
||||
error : function() {
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
// 更新表单中的多边形点位置数据
|
||||
function updatePolygonPoints() {
|
||||
var polygons = map.getOverlays(BMap.Polygon);
|
||||
for (var i = 0; i < polygons.length; i++) {
|
||||
// 确保当前对象是一个多边形对象
|
||||
if (polygons[i] instanceof BMap.Polygon) {
|
||||
var points = polygons[i].getPath();
|
||||
var bdShape = [];
|
||||
for (var j = 0; j < points.length; j++) {
|
||||
var location = [];
|
||||
location.push(points[j].lng);
|
||||
location.push(points[j].lat);
|
||||
bdShape.push(location);
|
||||
}
|
||||
// 将更新后的点位置数据更新到表单中
|
||||
$("#bdShape").val(bdShape.join(";"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
@@ -1052,8 +1052,8 @@
|
||||
<script type="text/javascript" src="../js/kindeditor-4.0/lang/zh_CN.js"></script>
|
||||
<script type="text/javascript" src="../js/numValidator.js"></script>
|
||||
<#--<script language="javascript" src="http://app.mapabc.com/apis?t=javascriptmap&v=3.1.1&key=b0a7db0b3a30f944a21c3682064dc70ef5b738b062f6479a5eca39725798b1ee300bd8d5de3a4ae3"></script>-->
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="http://webapi.amap.com/maps?v=1.4.3&key=65d0404c8a96fc2a630d9621c10c25bf"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://webapi.amap.com/maps?v=1.4.3&key=65d0404c8a96fc2a630d9621c10c25bf"></script>
|
||||
<script type="text/javascript">
|
||||
var flag=false;
|
||||
$(document).ready(function(){
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
<@menuitem uri='/agentMaster/listMaster' text="中介公司" />
|
||||
<@menuitem uri='/mortgageInfo/list' text="抵押品" />
|
||||
<@menuitem uri='/developmentZone/list' text="开发区" />
|
||||
<@menuitem uri='/township/list' text="乡镇街道" />
|
||||
</ul>
|
||||
</li>
|
||||
<li class="menu">
|
||||
|
||||
@@ -929,7 +929,7 @@
|
||||
</div><!-- /container -->
|
||||
|
||||
<div style="bottom: 1px;left: 45%;position: fixed;"><button type="button" id="editDist" class="btn btn-default btn-info form-control" style="width:200px;"> 保 存 </button></div>
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript">
|
||||
|
||||
$("#region").change(function(){
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
</style>
|
||||
<#include "/common_js.ftl" />
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript">
|
||||
$(document).ready(function(){
|
||||
$("#queryBtn").click(function(){
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
|
||||
</style>
|
||||
<#include "/common_js.ftl" />
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript">
|
||||
$(document).ready(function(){
|
||||
$("#queryBtn").click(function(){
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
package com.fangjia.dal.mongo.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDao;
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2024/6/11
|
||||
*/
|
||||
public interface AliasFangDao extends MongoDao<AliasFang> {
|
||||
|
||||
List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names);
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.dal.mongo.dao;
|
||||
|
||||
import com.fangjia.dal.MongoDaoImpl;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
import com.fangjia.dal.mongo.model.ConsumerAddress;
|
||||
import com.mongodb.BasicDBList;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.DBObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.sothis.core.util.CollectionUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class AliasFangDaoImpl extends MongoDaoImpl<AliasFang> implements AliasFangDao{
|
||||
|
||||
@Override
|
||||
public List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names) {
|
||||
DBObject query = new BasicDBObject();
|
||||
if (StringUtils.isNotBlank(city)) {
|
||||
query.put("city", city);
|
||||
}
|
||||
if (StringUtils.isNotBlank(region)) {
|
||||
query.put("region", region);
|
||||
}
|
||||
// db.alias_fang.find({'city': 'ÖØÇì', 'region': 'Óå±±', $or: [{'name': 'ÖйúĦ'}, {'alias': 'ÖйúĦ'}]})
|
||||
if(CollectionUtils.isNotEmpty(names)) {
|
||||
BasicDBList dbList = new BasicDBList();
|
||||
dbList.add(new BasicDBObject("name", new BasicDBObject("$in", names)));
|
||||
dbList.add(new BasicDBObject("alias", new BasicDBObject("$in", names)));
|
||||
query.put("$or", dbList);
|
||||
}
|
||||
List<AliasFang> objs = find(query, null, null, 0, Integer.MAX_VALUE);
|
||||
return objs;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package com.fangjia.dal.mongo.model;
|
||||
|
||||
import com.fangjia.dal.MongoEntity;
|
||||
|
||||
import javax.persistence.*;
|
||||
|
||||
/**
|
||||
* ËÑ·¿Ãû³ÆÓë±ðÃû¹ØÏµ
|
||||
*/
|
||||
@Entity
|
||||
@Table(name = "business.alias_fang")
|
||||
public class AliasFang implements MongoEntity {
|
||||
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String name;
|
||||
private String alias;
|
||||
|
||||
@Override
|
||||
@Id
|
||||
@GeneratedValue
|
||||
@Column(name = "_id", unique = true, nullable = false)
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
@Column(name = "city")
|
||||
public String getCity() {
|
||||
return city;
|
||||
}
|
||||
|
||||
public void setCity(String city) {
|
||||
this.city = city;
|
||||
}
|
||||
|
||||
@Column(name = "region")
|
||||
public String getRegion() {
|
||||
return region;
|
||||
}
|
||||
|
||||
public void setRegion(String region) {
|
||||
this.region = region;
|
||||
}
|
||||
|
||||
@Column(name = "name")
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
@Column(name = "alias")
|
||||
public String getAlias() {
|
||||
return alias;
|
||||
}
|
||||
|
||||
public void setAlias(String alias) {
|
||||
this.alias = alias;
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,12 @@ public class DevelopmentZone implements MongoEntity {
|
||||
private Map<String, Object> bdShape;
|
||||
|
||||
private String block;
|
||||
private String operator;
|
||||
private String cityName;
|
||||
private String district;
|
||||
|
||||
//增加别名字段
|
||||
private List<String> alias;
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.AUTO)
|
||||
@@ -170,4 +176,40 @@ public class DevelopmentZone implements MongoEntity {
|
||||
public void setBlock(String block) {
|
||||
this.block = block;
|
||||
}
|
||||
|
||||
@Column(name = "operator")
|
||||
public String getOperator() {
|
||||
return operator;
|
||||
}
|
||||
|
||||
public void setOperator(String operator) {
|
||||
this.operator = operator;
|
||||
}
|
||||
|
||||
@Column(name = "city")
|
||||
public String getCityName() {
|
||||
return cityName;
|
||||
}
|
||||
|
||||
public void setCityName(String cityName) {
|
||||
this.cityName = cityName;
|
||||
}
|
||||
|
||||
@Column(name = "district")
|
||||
public String getDistrict() {
|
||||
return district;
|
||||
}
|
||||
|
||||
public void setDistrict(String district) {
|
||||
this.district = district;
|
||||
}
|
||||
|
||||
@Column(name = "alias")
|
||||
public List<String> getAlias() {
|
||||
return alias;
|
||||
}
|
||||
|
||||
public void setAlias(List<String> alias) {
|
||||
this.alias = alias;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,7 +187,7 @@ public class District implements SeaweedEntity {
|
||||
private double score;
|
||||
// 小区性质
|
||||
private String landProperty;
|
||||
private String masterDistrictNname;
|
||||
private String masterDistrictName;
|
||||
private String ip;
|
||||
private String addrType;
|
||||
private boolean selected;
|
||||
@@ -309,12 +309,12 @@ public class District implements SeaweedEntity {
|
||||
}
|
||||
|
||||
@Column(name = "master_district_name")
|
||||
public String getMasterDistrictNname() {
|
||||
return masterDistrictNname;
|
||||
public String getMasterDistrictName() {
|
||||
return masterDistrictName;
|
||||
}
|
||||
|
||||
public void setMasterDistrictNname(String masterDistrictNname) {
|
||||
this.masterDistrictNname = masterDistrictNname;
|
||||
public void setMasterDistrictName(String masterDistrictName) {
|
||||
this.masterDistrictName = masterDistrictName;
|
||||
}
|
||||
|
||||
public void setCategory(String category) {
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<link href="../style/bootstrap/css/bootstrap.min.css" rel="stylesheet" media="screen">
|
||||
<link href="//cdn.bootcss.com/bootstrap/3.3.5/css/bootstrap-theme.min.css" rel="stylesheet" id="bs-theme-stylesheet">
|
||||
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div>
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
<link href="../style/bootstrap/css/bootstrap.min.css" rel="stylesheet" media="screen">
|
||||
<link href="//cdn.bootcss.com/bootstrap/3.3.5/css/bootstrap-theme.min.css" rel="stylesheet" id="bs-theme-stylesheet">
|
||||
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div>
|
||||
|
||||
+329
-1
@@ -14,11 +14,31 @@ import com.fangjia.open.util.CommonDtoUtils;
|
||||
import com.fangjia.service.bo.DistrictStatBo;
|
||||
import com.fangjia.service.datastore.DistrictService;
|
||||
import com.fangjia.service.district.DistrictStatService;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import sun.misc.BASE64Encoder;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.Authenticator;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.PasswordAuthentication;
|
||||
import java.net.Proxy;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.Stack;
|
||||
|
||||
/**
|
||||
* 地址治理
|
||||
@@ -358,8 +378,115 @@ public class AddrSvcController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
private static final Map<String, String> areaMap = new HashMap<String, String>() {
|
||||
{
|
||||
// 添加键值对到 Map 中
|
||||
put("111", "主城区");
|
||||
put("112", "城乡结合区");
|
||||
put("121", "镇中心区");
|
||||
put("122", "镇乡结合区");
|
||||
put("123", "特殊区域");
|
||||
put("210", "乡中心区");
|
||||
put("220", "村庄");
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 涉农识别-新规则
|
||||
* 涉农识别-新规则-南京银行
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object njcbVillageIdentifyAction(AddressModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
|
||||
String address = model.getAddress();
|
||||
params.put("address", address);
|
||||
String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null;
|
||||
params.put("type", type);
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8");
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v6/villageIdentify", params, "UTF-8");
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("msgCode");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if (json.containsKey("result")) {
|
||||
//我们的结果
|
||||
JSONObject result = json.getJSONObject("result");
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else {
|
||||
try {
|
||||
//如果是江苏省或者浙江省 调用齐梁的结果
|
||||
if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) {
|
||||
//结果比较,如果涉农结果不一致
|
||||
//调用齐梁 type :1 对公 0 对私
|
||||
//我们 type :1 对私 2 对共
|
||||
String qltype = "2".equals(type) ? "1" : "0";
|
||||
Map<String, String> map = senRequest(address, qltype);
|
||||
// 涉农结果不一样
|
||||
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
|
||||
//城市不一样 用我们
|
||||
if (result.containsKey("city")) {
|
||||
if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 涉农识别-新规则-基础版本
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
@@ -372,6 +499,207 @@ public class AddrSvcController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
|
||||
String address = model.getAddress();
|
||||
params.put("address", address);
|
||||
String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null;
|
||||
params.put("type", type);
|
||||
params.put("showFullValue", "2");
|
||||
// params.put("showTestValue", "2");
|
||||
Integer radius = model.getRadius();
|
||||
params.put("radius", (radius != null && radius != 0) ? radius.toString() : null);
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
if (json != null) {
|
||||
int code = json.getInt("msgCode");
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if (json.containsKey("result")) {
|
||||
//我们的结果
|
||||
JSONObject result = json.getJSONObject("result");
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else {
|
||||
try {
|
||||
//如果是江苏省或者浙江省 调用齐梁的结果
|
||||
if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) {
|
||||
|
||||
//结果比较,如果涉农结果不一致
|
||||
//调用齐梁 type :1 对公 0 对私
|
||||
//我们 type :1 对私 2 对共
|
||||
String qltype = "2".equals(type) ? "1" : "0";
|
||||
Map<String, String> map = senRequest(address, qltype);
|
||||
// 涉农结果不一样
|
||||
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
|
||||
//有市辖建制镇, 或者城市不一样 用我们
|
||||
if (result.containsKey("city")) {
|
||||
if ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
private static Map<String, String> senRequest(String address, String type) throws IOException {
|
||||
String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
String proxyHost = "tunnel2.qg.net";
|
||||
int proxyPort = 14186;
|
||||
final String proxyUsername = "113BBC06";
|
||||
final String proxyPassword = "3B2FBD05B0AF";
|
||||
String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}";
|
||||
String authKey = proxyUsername;
|
||||
String password = proxyPassword;
|
||||
String auth = authKey + ":" + password;
|
||||
String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8")));
|
||||
|
||||
CloseableHttpClient httpClient = HttpClients.createDefault();
|
||||
HttpPost httpPost = new HttpPost(targetURL);
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
Authenticator.setDefault(new Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray());
|
||||
}
|
||||
});
|
||||
httpPost.setHeader("Content-Type", "application/json");
|
||||
httpPost.setHeader("Accept", "*/*");
|
||||
httpPost.setHeader("Host", "146.56.217.44:9010");
|
||||
httpPost.setHeader("Proxy-Authorization", authHeader);
|
||||
StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON);
|
||||
httpPost.setEntity(stringEntity);
|
||||
|
||||
CloseableHttpResponse httpResponse = null;
|
||||
int retryCount = 2; // 设置重试次数
|
||||
int retryDelay = 10; // 设置重试延迟时间(毫秒)
|
||||
boolean success = false;
|
||||
for (int i = 0; i < retryCount; i++) {
|
||||
try {
|
||||
httpResponse = httpClient.execute(httpPost);
|
||||
success = true;
|
||||
break;
|
||||
} catch (IOException e) {
|
||||
System.err.println("代理请求失败,进行重试...");
|
||||
try {
|
||||
Thread.sleep(retryDelay);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (success) {
|
||||
HttpEntity responseEntity = httpResponse.getEntity();
|
||||
String responseString = EntityUtils.toString(responseEntity);
|
||||
|
||||
// 使用 Gson 解析 JSON 字符串
|
||||
Gson gson = new Gson();
|
||||
JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class);
|
||||
|
||||
|
||||
// 创建 Map 存储各个值
|
||||
Map<String, String> dataMap = new HashMap<String, String>();
|
||||
|
||||
// 获取各个值并放入 Map
|
||||
// dataMap.put("code", jsonObject.get("code").getAsString());
|
||||
// dataMap.put("msg", jsonObject.get("msg").getAsString());
|
||||
JsonElement dataElement = jsonObject.get("data");
|
||||
if (dataElement != null && !dataElement.isJsonNull() && dataElement.isJsonObject()) {
|
||||
JsonObject data = dataElement.getAsJsonObject();
|
||||
// 在这里继续处理 data 对象
|
||||
|
||||
if (data != null && !data.isJsonNull()) {
|
||||
|
||||
// dataMap.put("version", data.get("version").getAsString());
|
||||
dataMap.put("province", data.get("province").getAsString());
|
||||
dataMap.put("city", data.get("city").getAsString());
|
||||
dataMap.put("district", data.get("district").getAsString());
|
||||
dataMap.put("town", data.get("town").getAsString());
|
||||
dataMap.put("committee", data.get("village").getAsString());
|
||||
dataMap.put("provinceCode", data.get("provinceCode").getAsString());
|
||||
dataMap.put("cityCode", data.get("cityCode").getAsString());
|
||||
dataMap.put("districtCode", data.get("districtCode").getAsString());
|
||||
dataMap.put("townCode", data.get("townCode").getAsString());
|
||||
dataMap.put("committeeCode", data.get("villageCode").getAsString());
|
||||
dataMap.put("typeCode", data.get("areaCode").getAsString());
|
||||
dataMap.put("verifyFlag", "0".equals(data.get("farmer").getAsString()) ? "非农" : "农");
|
||||
dataMap.put("countyGov", "0".equals(data.get("isGovt").getAsString()) ? "否" : "是");
|
||||
// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空
|
||||
JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray();
|
||||
|
||||
if (indusParkList != null) {
|
||||
JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class);
|
||||
dataMap.put("devDistrictName", jsonObject1.get("indusParkName").getAsString());
|
||||
dataMap.put("devDistrict", "是");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
httpResponse.close();
|
||||
return dataMap;
|
||||
} else {
|
||||
System.err.println("重试次数已用完,请求失败。");
|
||||
}
|
||||
|
||||
httpClient.close();
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 涉农识别-新规则-标准版
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
*/
|
||||
public Object villageIdentifyStandardAction(AddressModel model) {
|
||||
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
|
||||
+40
-9
@@ -233,10 +233,13 @@ public class ComprehensiveQueryController {
|
||||
communityAvgPriceParams.put("city", model.getCity());
|
||||
communityAvgPriceParams.put("address", model.getAddress());
|
||||
communityAvgPriceParams.put("category", model.getCategory());
|
||||
communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap()));
|
||||
// communityAvgPriceParams.put("useMap", String.valueOf(model.isUseMap()));
|
||||
communityAvgPriceParams.put("suggestName", String.valueOf(model.getName()));
|
||||
communityAvgPriceParams.put("preciseLevel", "strict");
|
||||
communityAvgPriceParams.put("token", "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB");
|
||||
JSONObject matchResponse = null;
|
||||
try {
|
||||
matchResponse = httpRequest.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8");
|
||||
matchResponse = httpRequest.getJsonContent("http://open.fangjia.com/address/match", communityAvgPriceParams, "UTF-8");
|
||||
// matchResponse = httpRequest.getJsonContent("http://192.168.0.19:31080/FangJiaAddressService/v2/match", communityAvgPriceParams, "UTF-8");
|
||||
} catch (HttpRequestException e) {
|
||||
|
||||
@@ -256,9 +259,21 @@ public class ComprehensiveQueryController {
|
||||
DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(district));
|
||||
if (null != statBo) {
|
||||
if (statBo.getDistrictStat() != null) {
|
||||
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
|
||||
// 有2B价格,使用2B价格
|
||||
Integer avgPrice2b = statBo.getDistrictStat().getAvg_price_2b();
|
||||
if (avgPrice2b != null && avgPrice2b > 0) {
|
||||
dto.setAvgPrice(avgPrice2b);
|
||||
} else {
|
||||
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
|
||||
}
|
||||
} else if (statBo.getVillaDistrictStat() != null) {
|
||||
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
|
||||
// 有2B价格,使用2B价格
|
||||
Integer avgPrice2b = statBo.getVillaDistrictStat().getAvg_price_2b();
|
||||
if (avgPrice2b != null && avgPrice2b > 0) {
|
||||
dto.setAvgPrice(avgPrice2b);
|
||||
} else {
|
||||
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
@@ -638,10 +653,18 @@ public class ComprehensiveQueryController {
|
||||
if (evaluation != null && evaluation.getAvgPrice() != null && evaluation.getAvgPrice() > 0) {
|
||||
int factorPrice = evaluation.getAvgPrice();
|
||||
int avgPrice = evaluateFactorService.recompose(district.getCity(), district.getRegion(), evaluation.getArea(), factorPrice, orgAvgPrice);
|
||||
Integer minPrice = (int) (evaluation.getAvgPrice() * 0.95);
|
||||
Integer maxPrice = (int) (evaluation.getAvgPrice() * 1.05);
|
||||
// 有2B价格用2B价格,没有用2C
|
||||
Integer minPrice = 0;
|
||||
Integer maxPrice = 0;
|
||||
if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() > 0) {
|
||||
minPrice = (int) (evaluation.getAvgPrice2b() * 0.95);
|
||||
maxPrice = (int) (evaluation.getAvgPrice2b() * 1.05);
|
||||
} else {
|
||||
minPrice = (int) (evaluation.getAvgPrice() * 0.95);
|
||||
maxPrice = (int) (evaluation.getAvgPrice() * 1.05);
|
||||
}
|
||||
// 偏离度超过5%不进行调整
|
||||
if (minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||
if (minPrice != 0 && maxPrice != 0 && minPrice <= avgPrice && avgPrice <= maxPrice) {
|
||||
evaluation.setAvgPrice(avgPrice);
|
||||
evaluation.setBigTotalPrice((long) (avgPrice * evaluation.getArea()));
|
||||
}
|
||||
@@ -657,7 +680,11 @@ public class ComprehensiveQueryController {
|
||||
districtPriceMap.put("district", district.getRegion());
|
||||
districtPriceMap.put("name", district.getName());
|
||||
districtPriceMap.put("totalPrice", evaluation.getBigTotalPrice());
|
||||
districtPriceMap.put("avgPrice", evaluation.getAvgPrice());
|
||||
if (evaluation.getAvgPrice2b() != null && evaluation.getAvgPrice2b() != 0) {
|
||||
districtPriceMap.put("avgPrice", evaluation.getAvgPrice2b());
|
||||
} else {
|
||||
districtPriceMap.put("avgPrice", evaluation.getAvgPrice());
|
||||
}
|
||||
|
||||
// 根据城市等 级设置D级可信度
|
||||
ApiUserAuthority apiUserAuthority = apiUserAuthorityService
|
||||
@@ -882,7 +909,11 @@ public class ComprehensiveQueryController {
|
||||
otherPriceRets = new HashMap();
|
||||
}
|
||||
|
||||
otherPriceRets.put("fjw_price", evaluation.getAvgPrice());
|
||||
if (evaluation.getAvgPrice2b()!=null&&evaluation.getAvgPrice2b() > 0){
|
||||
otherPriceRets.put("fjw_price", evaluation.getAvgPrice2b());
|
||||
}else {
|
||||
otherPriceRets.put("fjw_price", evaluation.getAvgPrice());
|
||||
}
|
||||
Map<String, Object> levelMap = getPriceLevel(otherPriceRets, evaluation.getCredit());
|
||||
Long lastPrice = getLastPrice(levelMap, otherPriceRets, evaluation, dCredit);
|
||||
|
||||
|
||||
@@ -55,38 +55,44 @@ public class Controller {
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 缓存刷新接口
|
||||
* @return
|
||||
*/
|
||||
@Authority(role = Role.NONE)
|
||||
public Object refreshCacheAction(RefreshCacheModel model){
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
Token token = FangjiaApiUtil.parse2Token(model.getToken());
|
||||
if(token == null){
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
|
||||
return mav;
|
||||
}
|
||||
final ThreadLocalData threadLocalData = ThreadLocalData.getInstance();
|
||||
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
|
||||
// 在验证appKey
|
||||
ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId());
|
||||
if(apiUser == null){
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
|
||||
return mav;
|
||||
}
|
||||
try {
|
||||
apiUserAuthorityService.findByUserId(apiUser.getId());
|
||||
if(StringUtils.isNotBlank(model.getViewConfigUrl())){
|
||||
apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl());
|
||||
}
|
||||
} catch (IllegalAccessException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功"));
|
||||
return mav;
|
||||
}
|
||||
/**
|
||||
* 缓存刷新接口
|
||||
*
|
||||
* @return
|
||||
*/
|
||||
@Authority(role = Role.NONE)
|
||||
public Object refreshCacheAction(RefreshCacheModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
Token token = FangjiaApiUtil.parse2Token(model.getToken());
|
||||
if (token == null) {
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
|
||||
return mav;
|
||||
}
|
||||
final ThreadLocalData threadLocalData = ThreadLocalData.getInstance();
|
||||
threadLocalData.put(ThreadLocalKey.DisableCache, "1");
|
||||
// 在验证appKey
|
||||
ApiUser apiUser = this.apiUserService.findApiUserById(token.getUserId());
|
||||
if (apiUser == null) {
|
||||
mav.setModel(BasicModel.buildUnAuthorizedModel().setMsg("非法的Token"));
|
||||
return mav;
|
||||
}
|
||||
try {
|
||||
// 获取用户
|
||||
User user = userService.findById(apiUser.getId());
|
||||
if (null != user && StringUtils.isNotBlank(user.getUserName())) {
|
||||
userService.findByUserName(user.getUserName());
|
||||
}
|
||||
apiUserAuthorityService.findByUserId(apiUser.getId());
|
||||
if (StringUtils.isNotBlank(model.getViewConfigUrl())) {
|
||||
apiUserAuthorityService.findViewConfig(apiUser.getId(), model.getViewConfigUrl());
|
||||
}
|
||||
} catch (IllegalAccessException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
mav.setModel(BasicModel.buildOKModel().setMsg("缓存刷新成功"));
|
||||
return mav;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+192
-5
@@ -6,9 +6,28 @@ import com.fangjia.open.interceptors.Role;
|
||||
import com.fangjia.open.model.AddressModel;
|
||||
import com.fangjia.open.model.BasicModel;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
import com.google.gson.Gson;
|
||||
import com.google.gson.JsonArray;
|
||||
import com.google.gson.JsonElement;
|
||||
import com.google.gson.JsonObject;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.http.HttpEntity;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPost;
|
||||
import org.apache.http.entity.ContentType;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import sun.misc.BASE64Encoder;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.Authenticator;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.PasswordAuthentication;
|
||||
import java.net.Proxy;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@@ -18,8 +37,21 @@ import java.util.Map;
|
||||
@Authority(role = Role.USER)
|
||||
public class GdBankController {
|
||||
|
||||
private static final Map<String, String> areaMap = new HashMap<String, String>() {
|
||||
{
|
||||
// 添加键值对到 Map 中
|
||||
put("111", "主城区");
|
||||
put("112", "城乡结合区");
|
||||
put("121", "镇中心区");
|
||||
put("122", "镇乡结合区");
|
||||
put("123", "特殊区域");
|
||||
put("210", "乡中心区");
|
||||
put("220", "村庄");
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* 涉农识别-新规
|
||||
* 涉农识别-新规则-光大银行
|
||||
*
|
||||
* @param model
|
||||
* @return
|
||||
@@ -31,12 +63,14 @@ public class GdBankController {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
|
||||
return mav;
|
||||
}
|
||||
String type = StringUtils.isNotBlank(model.getType()) ? model.getType() : null;
|
||||
String address = model.getAddress();
|
||||
|
||||
HttpRequest request = new HttpRequest();
|
||||
JSONObject json = null;
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
|
||||
params.put("type", StringUtils.isNotBlank(model.getType()) ? model.getType() : null);
|
||||
params.put("type", type);
|
||||
params.put("redis", (model.getRadius() != null && model.getRadius() != 0) ? model.getRadius().toString() : null);
|
||||
params.put("address", model.getAddress());
|
||||
params.put("showFullValue", "2");
|
||||
@@ -54,13 +88,63 @@ public class GdBankController {
|
||||
String msg = json.getString("msg");
|
||||
BasicModel basicModel = new BasicModel(code, msg);
|
||||
if (json.containsKey("result")) {
|
||||
//我们的结果
|
||||
JSONObject result = json.getJSONObject("result");
|
||||
if (("是").equals(result.get("countyLevelCity"))){
|
||||
if (("是").equals(result.get("countyLevelCity"))) {
|
||||
result.put("prefectureLevelCity", "否");
|
||||
} else if(("否").equals(result.get("countyLevelCity"))){
|
||||
} else if (("否").equals(result.get("countyLevelCity"))) {
|
||||
result.put("prefectureLevelCity", "是");
|
||||
}
|
||||
basicModel.setResult(result);
|
||||
// 如果是无效信息或者没有行政区划,则用我们的
|
||||
if (result.containsKey("addrVerify") && ("无效".equals(result.getString("addrVerify")) || "没有行政区划".equals(result.getString("addrVerify")))) {
|
||||
basicModel.setResult(result);
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
} else if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
try {
|
||||
//如果是江苏省或者浙江省 调用齐梁的结果
|
||||
if ("江苏省".equals(result.getString("province")) || "浙江省".equals(result.getString("province"))) {
|
||||
|
||||
//结果比较,如果涉农结果不一致
|
||||
//调用齐梁 type :1 对公 0 对私
|
||||
//我们 type :1 对私 2 对共
|
||||
String qltype = "2".equals(type) ? "1" : "0";
|
||||
Map<String, String> map = senRequest(address, qltype);
|
||||
// 涉农结果不一样
|
||||
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
|
||||
//有市辖建制镇, 或者城市不一样 用我们
|
||||
if (result.containsKey("city") && ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city"))
|
||||
&& StringUtils.isNotBlank(result.getString("city"))))) {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
if (("是").equals(result.get("countyLevelCity"))) {
|
||||
map.put("prefectureLevelCity", "否");
|
||||
} else if (("否").equals(result.get("countyLevelCity"))) {
|
||||
map.put("prefectureLevelCity", "是");
|
||||
}
|
||||
map.put("addrVerify", "地址有效");
|
||||
if (result.containsKey("countyLevelCity")) {
|
||||
map.put("countyLevelCity", result.getString("countyLevelCity"));
|
||||
}
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
} else {
|
||||
basicModel.setResult(result);
|
||||
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
basicModel.setResult(result);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
@@ -68,4 +152,107 @@ public class GdBankController {
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
private static Map<String, String> senRequest(String address, String type) throws IOException {
|
||||
String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
String proxyHost = "tunnel2.qg.net";
|
||||
int proxyPort = 14186;
|
||||
final String proxyUsername = "113BBC06";
|
||||
final String proxyPassword = "3B2FBD05B0AF";
|
||||
String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}";
|
||||
String authKey = proxyUsername;
|
||||
String password = proxyPassword;
|
||||
String auth = authKey + ":" + password;
|
||||
String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8")));
|
||||
|
||||
CloseableHttpClient httpClient = HttpClients.createDefault();
|
||||
HttpPost httpPost = new HttpPost(targetURL);
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
Authenticator.setDefault(new Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray());
|
||||
}
|
||||
});
|
||||
httpPost.setHeader("Content-Type", "application/json");
|
||||
httpPost.setHeader("Accept", "*/*");
|
||||
httpPost.setHeader("Host", "146.56.217.44:9010");
|
||||
httpPost.setHeader("Proxy-Authorization", authHeader);
|
||||
StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON);
|
||||
httpPost.setEntity(stringEntity);
|
||||
|
||||
CloseableHttpResponse httpResponse = null;
|
||||
int retryCount = 2; // 设置重试次数
|
||||
int retryDelay = 10; // 设置重试延迟时间(毫秒)
|
||||
boolean success = false;
|
||||
for (int i = 0; i < retryCount; i++) {
|
||||
try {
|
||||
httpResponse = httpClient.execute(httpPost);
|
||||
success = true;
|
||||
break;
|
||||
} catch (IOException e) {
|
||||
System.err.println("代理请求失败,进行重试...");
|
||||
try {
|
||||
Thread.sleep(retryDelay);
|
||||
} catch (InterruptedException ex) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (success) {
|
||||
HttpEntity responseEntity = httpResponse.getEntity();
|
||||
String responseString = EntityUtils.toString(responseEntity);
|
||||
|
||||
// 使用 Gson 解析 JSON 字符串
|
||||
Gson gson = new Gson();
|
||||
JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class);
|
||||
|
||||
|
||||
// 创建 Map 存储各个值
|
||||
Map<String, String> dataMap = new HashMap<String, String>();
|
||||
|
||||
// 获取各个值并放入 Map
|
||||
// dataMap.put("code", jsonObject.get("code").getAsString());
|
||||
// dataMap.put("msg", jsonObject.get("msg").getAsString());
|
||||
JsonElement dataElement = jsonObject.get("data");
|
||||
if (dataElement != null && !dataElement.isJsonNull() && dataElement.isJsonObject()) {
|
||||
JsonObject data = dataElement.getAsJsonObject();
|
||||
// 在这里继续处理 data 对象
|
||||
|
||||
if (data != null && !data.isJsonNull()) {
|
||||
|
||||
// dataMap.put("version", data.get("version").getAsString());
|
||||
dataMap.put("province", data.get("province").getAsString());
|
||||
dataMap.put("city", data.get("city").getAsString());
|
||||
dataMap.put("district", data.get("district").getAsString());
|
||||
dataMap.put("town", data.get("town").getAsString());
|
||||
dataMap.put("committee", data.get("village").getAsString());
|
||||
dataMap.put("provinceCode", data.get("provinceCode").getAsString());
|
||||
dataMap.put("cityCode", data.get("cityCode").getAsString());
|
||||
dataMap.put("districtCode", data.get("districtCode").getAsString());
|
||||
dataMap.put("townCode", data.get("townCode").getAsString());
|
||||
dataMap.put("committeeCode", data.get("villageCode").getAsString());
|
||||
dataMap.put("typeCode", data.get("areaCode").getAsString());
|
||||
dataMap.put("verifyFlag", "0".equals(data.get("farmer").getAsString()) ? "非农" : "农");
|
||||
dataMap.put("countyGov", "0".equals(data.get("isGovt").getAsString()) ? "否" : "是");
|
||||
// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空
|
||||
JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray();
|
||||
|
||||
if (indusParkList != null) {
|
||||
JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class);
|
||||
dataMap.put("devDistrictName", jsonObject1.get("indusParkName").getAsString());
|
||||
dataMap.put("devDistrict", "是");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
httpResponse.close();
|
||||
return dataMap;
|
||||
} else {
|
||||
System.err.println("重试次数已用完,请求失败。");
|
||||
}
|
||||
|
||||
httpClient.close();
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
+107
-29
@@ -10,17 +10,7 @@ import java.net.URLEncoder;
|
||||
import java.nio.charset.Charset;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Calendar;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
@@ -30,6 +20,7 @@ import javax.validation.Valid;
|
||||
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.dal.*;
|
||||
import com.fangjia.dal.mongo.dao.AliasFangDao;
|
||||
import com.fangjia.dal.mongo.model.*;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityBasePrice;
|
||||
import com.fangjia.dal.mongo.open.model.CommunityPoi;
|
||||
@@ -39,6 +30,7 @@ import com.fangjia.dal.seawater.model.Seawater;
|
||||
import com.fangjia.open.dto.*;
|
||||
import com.fangjia.open.dto.addressGovernance.StructureDto;
|
||||
import com.fangjia.open.util.CommonDtoUtils;
|
||||
import com.fangjia.service.address.AddressParserUtil;
|
||||
import com.fangjia.service.district.*;
|
||||
import com.fangjia.service.mongo.fangjia.*;
|
||||
import com.fangjia.service.open.*;
|
||||
@@ -138,7 +130,6 @@ import com.mongodb.DBObject;
|
||||
|
||||
@Authority(role = Role.USER)
|
||||
public class PropertyController {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
|
||||
private DistrictService districtService;
|
||||
private AddressParser addressParser;
|
||||
@@ -186,7 +177,7 @@ public class PropertyController {
|
||||
|
||||
private FileDao fileDao = FileDao.getInstance();
|
||||
private static final int DEFAULT_PAGE_SIZE = 20;
|
||||
private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias" };
|
||||
private static final String[] PROPERTY_SEARCH_API_FIELDS = { "address", "city", "region", "name", "lat2", "lng2","lat", "lng", "block", "estateType2", "addrAlias", "alias" };
|
||||
static final int BAI_DU_GEO = 1;// 百度系坐标
|
||||
static final int GAO_DE_GEO = 2;// 高德系坐标
|
||||
|
||||
@@ -196,6 +187,7 @@ public class PropertyController {
|
||||
private DistrictRoomSqmetreRentService districtRoomSqmetreRentService;
|
||||
private RentDealPriceService rentDealPriceService;
|
||||
private EvaluateRatioService evaluateRatioService;
|
||||
private AliasFangService aliasFangService;
|
||||
|
||||
private static final int EVALUATE_SHOW_DETAIL = 1;
|
||||
private static final Map<String, String> PROPERTY_TYPES = new HashMap<String, String>() {
|
||||
@@ -609,6 +601,25 @@ public class PropertyController {
|
||||
dto.setBlock(district.getBlock());
|
||||
}
|
||||
dto.setPropertyType(district.getEstateType2());
|
||||
if (CollectionUtils.isNotEmpty(district.getAlias())) {
|
||||
// 将最相近的别名输出给用户
|
||||
String alias = null;
|
||||
double similarity = 0;
|
||||
if (StringUtils.isNotBlank(model.getKeyword())) {
|
||||
for (String mappingAlias : district.getAlias()) {
|
||||
double curr = AddressParserUtil.getSimilarityRatio(mappingAlias, model.getKeyword());
|
||||
if (curr > similarity ) {
|
||||
similarity = curr;
|
||||
alias = mappingAlias;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (StringUtils.isNotBlank(alias)) {
|
||||
dto.setAlias(alias);
|
||||
} else {
|
||||
dto.setAlias(district.getAlias().get(0));
|
||||
}
|
||||
}
|
||||
list.add(dto);
|
||||
}
|
||||
|
||||
@@ -2516,7 +2527,7 @@ public class PropertyController {
|
||||
model.setMax(100);
|
||||
}
|
||||
|
||||
List<String> districtNames = new ArrayList<String>();
|
||||
Set<String> districtNames = new HashSet<String>();
|
||||
if (null == district) {
|
||||
districtNames.add(model.getName());
|
||||
} else {
|
||||
@@ -2524,14 +2535,56 @@ public class PropertyController {
|
||||
if (district.getAlias() != null) {
|
||||
districtNames.addAll(district.getAlias());
|
||||
}
|
||||
}
|
||||
List<DealPrice> dealPrices = null;
|
||||
for (String name : districtNames) {
|
||||
dealPrices = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model);
|
||||
if (!dealPrices.isEmpty()) {
|
||||
break;
|
||||
final String masterDistrictName = district.getMasterDistrictName();
|
||||
if (StringUtils.isNotBlank(masterDistrictName)) {
|
||||
List<District> districts;
|
||||
if ("master".equals(masterDistrictName)) {
|
||||
// 所有的分期数据
|
||||
districts = districtService.findSlaves(model.getCity(), model.getDistrict(), model.getName());
|
||||
} else {
|
||||
// 根据主小区名查询所有名称
|
||||
districts = districtService.findSlaves(model.getCity(), model.getDistrict(), masterDistrictName);
|
||||
}
|
||||
if (CollectionUtils.isNotEmpty(districts)) {
|
||||
for (District districtTemp : districts) {
|
||||
districtNames.add(districtTemp.getName());
|
||||
}
|
||||
}
|
||||
|
||||
// districtNames 不包含 主小区名,加进去
|
||||
if (!districtNames.contains(masterDistrictName)){
|
||||
districtNames.add(masterDistrictName);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
// 将搜房中的别名与名称加入查询集合
|
||||
try {
|
||||
List<AliasFang> aliasFangs = aliasFangService.findFangByNameAndAlias(model.getCity(), model.getDistrict(), districtNames);
|
||||
Set<String> aliasSet = new HashSet<String>();
|
||||
for (AliasFang aliasFang : aliasFangs) {
|
||||
if (StringUtils.isNotBlank(aliasFang.getName())) {
|
||||
aliasSet.add(aliasFang.getName());
|
||||
}
|
||||
if (StringUtils.isNotBlank(aliasFang.getAlias())) {
|
||||
aliasSet.add(aliasFang.getAlias());
|
||||
}
|
||||
}
|
||||
districtNames.addAll(aliasSet);
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
}
|
||||
|
||||
|
||||
// List<DealPrice> dealPrices = new ArrayList<DealPrice>();
|
||||
// 将主名与映射别名查得成交加入 存在不能查出最新成交问题
|
||||
// for (String name : districtNames) {
|
||||
// List<DealPrice> dealPricesTempList = findDealPricesByName(model.getCity(), model.getDistrict(), name, model.getMax() * 5, model);
|
||||
// if (CollectionUtils.isNotEmpty(dealPricesTempList)) {
|
||||
// dealPrices.addAll(dealPricesTempList);
|
||||
// }
|
||||
// }
|
||||
List<DealPrice> dealPrices = findDealPricesByNames(model.getCity(), model.getDistrict(), new ArrayList<String>(districtNames), model.getMax() * 5, model);
|
||||
if (model.getSize() != null) {
|
||||
Collections.sort(dealPrices, new Comparator<DealPrice>() {
|
||||
@Override
|
||||
@@ -2580,15 +2633,9 @@ public class PropertyController {
|
||||
}
|
||||
deal = new LinkedHashMap<String, Object>();
|
||||
dealPrice.setId(null);
|
||||
if (district != null) {
|
||||
deal.put("city", district.getCity());
|
||||
deal.put("district", district.getRegion());
|
||||
deal.put("name", district.getName());
|
||||
} else {
|
||||
deal.put("city", dealPrice.getCity());
|
||||
deal.put("district", dealPrice.getRegion());
|
||||
deal.put("name", dealPrice.getDistrictName());
|
||||
}
|
||||
deal.put("city", dealPrice.getCity());
|
||||
deal.put("district", dealPrice.getRegion());
|
||||
deal.put("name", dealPrice.getDistrictName());
|
||||
if (dealPrice.getArea() == null) {
|
||||
deal.put("size", "");
|
||||
} else {
|
||||
@@ -3501,6 +3548,33 @@ public class PropertyController {
|
||||
return null;
|
||||
}
|
||||
|
||||
private List<DealPrice> findDealPricesByNames(String city, String region, List<String> names, int max,
|
||||
DealPriceModel model) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("city", city));
|
||||
params.add(new QueryParam("region", region));
|
||||
params.add(new QueryParam("districtName", QueryParam.IN_OPERATOR, names));
|
||||
if (model.getRoom() != null) {
|
||||
params.add(new QueryParam("room", model.getRoom()));
|
||||
}
|
||||
if (model.getHall() != null) {
|
||||
params.add(new QueryParam("hall", model.getHall()));
|
||||
}
|
||||
// 18年之后抓住的数据没有score字段
|
||||
// params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l));
|
||||
List<DealPrice> dealPrices = dealPriceService.findDealPrices(params, max);
|
||||
List<DealPrice> distinctDealPrices = new ArrayList<DealPrice>();
|
||||
for (DealPrice dealPrice : dealPrices) {
|
||||
if (dealPrice.getAvgPrice() == null) {
|
||||
dealPrice.setAvgPrice((int) (dealPrice.getTotalPrice() / dealPrice.getArea()));
|
||||
}
|
||||
if (!checkExists(distinctDealPrices, dealPrice)) {
|
||||
distinctDealPrices.add(dealPrice);
|
||||
}
|
||||
}
|
||||
return distinctDealPrices;
|
||||
}
|
||||
|
||||
private List<DealPrice> findDealPricesByName(String city, String region, String name, int max,
|
||||
DealPriceModel model) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
@@ -5959,4 +6033,8 @@ public class PropertyController {
|
||||
public void setCommunityBasePriceService(CommunityBasePriceService communityBasePriceService) {
|
||||
this.communityBasePriceService = communityBasePriceService;
|
||||
}
|
||||
|
||||
public void setAliasFangService(AliasFangService aliasFangService) {
|
||||
this.aliasFangService = aliasFangService;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
//package com.fangjia.open.controllers;
|
||||
//
|
||||
//import com.google.gson.Gson;
|
||||
//import com.google.gson.JsonArray;
|
||||
//import com.google.gson.JsonObject;
|
||||
//import org.apache.http.HttpEntity;
|
||||
//import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
//import org.apache.http.client.methods.HttpPost;
|
||||
//import org.apache.http.entity.ContentType;
|
||||
//import org.apache.http.entity.StringEntity;
|
||||
//import org.apache.http.impl.client.CloseableHttpClient;
|
||||
//import org.apache.http.impl.client.HttpClients;
|
||||
//import org.apache.http.util.EntityUtils;
|
||||
//import sun.misc.BASE64Encoder;
|
||||
//
|
||||
//import java.io.IOException;
|
||||
//import java.net.Authenticator;
|
||||
//import java.net.InetSocketAddress;
|
||||
//import java.net.PasswordAuthentication;
|
||||
//import java.net.Proxy;
|
||||
//import java.nio.charset.Charset;
|
||||
//import java.nio.charset.StandardCharsets;
|
||||
//import java.util.Base64;
|
||||
//import java.util.HashMap;
|
||||
//import java.util.Map;
|
||||
//
|
||||
////public class Test {
|
||||
//// public static void main(String[] args) throws IOException {
|
||||
//// // 目标URL
|
||||
//// String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
////
|
||||
//// // 设置代理
|
||||
//// String proxyHost = "tunnel2.qg.net";
|
||||
//// int proxyPort = 14186;
|
||||
////
|
||||
//// // 设置代理用户名和密码
|
||||
//// final String proxyUsername = "113BBC06";
|
||||
//// final String proxyPassword = "3B2FBD05B0AF";
|
||||
////
|
||||
//// // 设置请求的JSON数据
|
||||
//// String address = "江苏省扬州市宝应县安宜镇宝应大道88号";
|
||||
//// String type = "1";
|
||||
//// String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}";
|
||||
////
|
||||
//// // 设置HTTP基本认证
|
||||
//// String authKey = proxyUsername;
|
||||
//// String password = proxyPassword;
|
||||
//// String auth = authKey + ":" + password;
|
||||
//// String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8")));
|
||||
////
|
||||
//// // 创建HTTP客户端
|
||||
//// CloseableHttpClient httpClient = HttpClients.createDefault();
|
||||
//// HttpPost httpPost = new HttpPost(targetURL);
|
||||
////
|
||||
//// // 设置代理
|
||||
//// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
////
|
||||
//// // 设置代理认证
|
||||
//// Authenticator.setDefault(new Authenticator() {
|
||||
//// protected PasswordAuthentication getPasswordAuthentication() {
|
||||
//// return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray());
|
||||
//// }
|
||||
//// });
|
||||
////
|
||||
//// // 设置请求头
|
||||
//// httpPost.setHeader("Content-Type", "application/json");
|
||||
//// httpPost.setHeader("Accept", "*/*");
|
||||
//// httpPost.setHeader("Host", "146.56.217.44:9010");
|
||||
//// httpPost.setHeader("Proxy-Authorization", authHeader);
|
||||
////
|
||||
//// // 设置请求体
|
||||
//// StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON);
|
||||
//// httpPost.setEntity(stringEntity);
|
||||
////
|
||||
//// // 发送POST请求
|
||||
//// CloseableHttpResponse httpResponse = httpClient.execute(httpPost);
|
||||
////
|
||||
//// // 获取响应
|
||||
//// HttpEntity responseEntity = httpResponse.getEntity();
|
||||
//// String responseString = EntityUtils.toString(responseEntity);
|
||||
//// System.out.println(responseString);
|
||||
////
|
||||
//// // 关闭连接
|
||||
//// httpResponse.close();
|
||||
//// httpClient.close();
|
||||
//// }
|
||||
////}
|
||||
//
|
||||
//public class Test {
|
||||
// public static void main(String[] args) throws IOException {
|
||||
// Map<String, String> map = senRequest("江苏省泰州市靖江市通江路37号", "1");
|
||||
// System.out.println(map);
|
||||
//
|
||||
// }
|
||||
//
|
||||
// private static Map<String, String> senRequest(String address, String type) throws IOException {
|
||||
// String targetURL = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
// String proxyHost = "tunnel2.qg.net";
|
||||
// int proxyPort = 14186;
|
||||
// final String proxyUsername = "113BBC06";
|
||||
// final String proxyPassword = "3B2FBD05B0AF";
|
||||
// String jsonData = "{\"address\":\"" + address + "\",\"type\":\"" + type + "\"}";
|
||||
// String authKey = proxyUsername;
|
||||
// String password = proxyPassword;
|
||||
// String auth = authKey + ":" + password;
|
||||
// String authHeader = "Basic " + new BASE64Encoder().encode(auth.getBytes(Charset.forName("UTF-8")));
|
||||
//
|
||||
// CloseableHttpClient httpClient = HttpClients.createDefault();
|
||||
// HttpPost httpPost = new HttpPost(targetURL);
|
||||
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
// Authenticator.setDefault(new Authenticator() {
|
||||
// protected PasswordAuthentication getPasswordAuthentication() {
|
||||
// return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray());
|
||||
// }
|
||||
// });
|
||||
// httpPost.setHeader("Content-Type", "application/json");
|
||||
// httpPost.setHeader("Accept", "*/*");
|
||||
// httpPost.setHeader("Host", "146.56.217.44:9010");
|
||||
// httpPost.setHeader("Proxy-Authorization", authHeader);
|
||||
// StringEntity stringEntity = new StringEntity(jsonData, ContentType.APPLICATION_JSON);
|
||||
// httpPost.setEntity(stringEntity);
|
||||
//
|
||||
// CloseableHttpResponse httpResponse = null;
|
||||
// int retryCount = 2; // 设置重试次数
|
||||
// int retryDelay = 10; // 设置重试延迟时间(毫秒)
|
||||
// boolean success = false;
|
||||
// for (int i = 0; i < retryCount; i++) {
|
||||
// try {
|
||||
// httpResponse = httpClient.execute(httpPost);
|
||||
// success = true;
|
||||
// break;
|
||||
// } catch (IOException e) {
|
||||
// System.err.println("代理请求失败,进行重试...");
|
||||
// try {
|
||||
// Thread.sleep(retryDelay);
|
||||
// } catch (InterruptedException ex) {
|
||||
// Thread.currentThread().interrupt();
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (success) {
|
||||
// HttpEntity responseEntity = httpResponse.getEntity();
|
||||
// String responseString = EntityUtils.toString(responseEntity);
|
||||
//
|
||||
// // 使用 Gson 解析 JSON 字符串
|
||||
// Gson gson = new Gson();
|
||||
// JsonObject jsonObject = gson.fromJson(responseString, JsonObject.class);
|
||||
//
|
||||
//
|
||||
// // 创建 Map 存储各个值
|
||||
// Map<String, String> dataMap = new HashMap<String, String>();
|
||||
//
|
||||
// // 获取各个值并放入 Map
|
||||
// dataMap.put("code", jsonObject.get("code").getAsString());
|
||||
// dataMap.put("msg", jsonObject.get("msg").getAsString());
|
||||
// JsonObject data = jsonObject.getAsJsonObject("data");
|
||||
// dataMap.put("version", data.get("version").getAsString());
|
||||
// dataMap.put("province", data.get("province").getAsString());
|
||||
// dataMap.put("city", data.get("city").getAsString());
|
||||
// dataMap.put("district", data.get("district").getAsString());
|
||||
// dataMap.put("town", data.get("town").getAsString());
|
||||
// dataMap.put("village", data.get("village").getAsString());
|
||||
// dataMap.put("provinceCode", data.get("provinceCode").getAsString());
|
||||
// dataMap.put("cityCode", data.get("cityCode").getAsString());
|
||||
// dataMap.put("districtCode", data.get("districtCode").getAsString());
|
||||
// dataMap.put("townCode", data.get("townCode").getAsString());
|
||||
// dataMap.put("villageCode", data.get("villageCode").getAsString());
|
||||
// dataMap.put("areaCode", data.get("areaCode").getAsString());
|
||||
// dataMap.put("farmer", data.get("farmer").getAsString());
|
||||
// dataMap.put("isGovt", data.get("isGovt").getAsString());
|
||||
// dataMap.put("govtAddr", data.get("govtAddr").isJsonNull() ? null : data.get("govtAddr").getAsString()); // 对于可能为 null 的值,需要判断是否为空
|
||||
// JsonArray indusParkList = data.get("indusParkList").isJsonNull() ? null : data.get("indusParkList").getAsJsonArray();
|
||||
// if (indusParkList != null) {
|
||||
// JsonObject jsonObject1 = gson.fromJson(indusParkList.get(0), JsonObject.class);
|
||||
// dataMap.put("devName",jsonObject1.get("indusParkName").getAsString());
|
||||
// dataMap.put("devLevel",jsonObject1.get("indusParkLevel").getAsString());
|
||||
// }
|
||||
//
|
||||
// httpResponse.close();
|
||||
// return dataMap;
|
||||
// } else {
|
||||
// System.err.println("重试次数已用完,请求失败。");
|
||||
// }
|
||||
//
|
||||
// httpClient.close();
|
||||
// return null;
|
||||
// }
|
||||
//}
|
||||
//
|
||||
@@ -15,6 +15,10 @@ public class PropertyDto {
|
||||
private String propertyType;
|
||||
private String block;
|
||||
private String distance;
|
||||
/**
|
||||
* 新增小区别名
|
||||
*/
|
||||
private String alias;
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
@@ -88,4 +92,12 @@ public class PropertyDto {
|
||||
public void setDistance(String distance) {
|
||||
this.distance = distance;
|
||||
}
|
||||
|
||||
public String getAlias() {
|
||||
return alias;
|
||||
}
|
||||
|
||||
public void setAlias(String alias) {
|
||||
this.alias = alias;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>城市地图</title>
|
||||
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="allmap" style="width: 100%;height: 100%;"></div>
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<title>地图</title>
|
||||
<link href="../style/bootstrap/css/bootstrap.min.css" rel="stylesheet" media="screen">
|
||||
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div>
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>学校地图</title>
|
||||
<script type="text/javascript" src="../js/jquery-1.9.1.js"></script>
|
||||
<script type="text/javascript" src="http://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
<script type="text/javascript" src="https://api.map.baidu.com/api?v=2.0&ak=3E63A744012f88c9a73a2f051aece58b"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div id="allmap" style="width: 100%;height: 100%;"></div>
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
// 分期分区与主小区映射关系表
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class FindStageDistrictBatch extends BaseBatch {
|
||||
|
||||
|
||||
+12
-8
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳2024年3月份二手房成交数据汇总.xls", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
@@ -75,18 +75,22 @@ for(def d : datas){
|
||||
map.area = d['F'].toString().toDouble()
|
||||
map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000)
|
||||
map.avg_price = (int)(d['H'].toString().toDouble())
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
def trade_date = d['J']
|
||||
if(trade_date){
|
||||
map.trade_date = trade_date
|
||||
}else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
if(d['I']){
|
||||
map.sell_price = (int)(d['I'].toString().toDouble()*10000)
|
||||
}
|
||||
|
||||
// def trade_date = d['J']
|
||||
// if(trade_date){
|
||||
// map.trade_date = DateUtils.parseFromTightByPattern(trade_date,'yyyy/MM/dd')
|
||||
// }else{
|
||||
def trade_date1 = DateUtils.getRandomDate("2024-03-01", "2024-03-31")
|
||||
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
|
||||
// }
|
||||
|
||||
map.source = "深房家族"
|
||||
map.create_date = new Date()
|
||||
map.m_date = new Date()
|
||||
println map
|
||||
DEAL_PRICE_COLL.save(map)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -23,7 +23,7 @@ class AddThirdPartyPrice {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def c : citys){
|
||||
println c +" "+cityCodeDatas.get(c)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(c, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(c)
|
||||
def rets = createDate(dists, distCodeMap)
|
||||
println dists.size() +" "+rets.size()
|
||||
|
||||
+1
-1
@@ -47,7 +47,7 @@ class BigdataBuildingBatch extends BaseBatch{
|
||||
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias","build_type","build_structure"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
def alias = d.alias;
|
||||
|
||||
+1
-1
@@ -58,7 +58,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
|
||||
|
||||
def ccbMaps = null
|
||||
|
||||
+37
-7
@@ -17,6 +17,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -61,13 +62,21 @@ class BigdataDealBatch extends BaseBatch{
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def date = DateUtils.parseFromTightByPattern("2024-5-21", "yyyy-MM-dd")
|
||||
def objectId = dateToObjectId(date)
|
||||
println objectId
|
||||
|
||||
//664a22000000000000000000 5-20
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
//
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
@@ -529,7 +538,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], null)
|
||||
println city +" 小区量:"+dists.size()
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
@@ -679,6 +688,16 @@ class BigdataDealBatch extends BaseBatch{
|
||||
return map
|
||||
}
|
||||
|
||||
def updateCrawlTradeDate(){
|
||||
def dateStr = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-20"
|
||||
def date = DateUtils.parseFromTightByPattern(dateStr, "yyyy-MM-dd")
|
||||
def objectId = dateToObjectId(date)
|
||||
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -2)
|
||||
def endDate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
println(objectId)
|
||||
CcbCommonUtils.updateTrades(startDate, endDate, objectId)
|
||||
}
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
@@ -816,4 +835,15 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def dateToObjectId(Date date) {
|
||||
// 获取时间戳(单位:秒)
|
||||
def timestamp = (date.time / 1000) as int
|
||||
def hexTimestamp = Integer.toHexString(timestamp)
|
||||
|
||||
// 创建 ObjectId 字符串:前4字节为时间戳,后8字节可以是任意值
|
||||
def objectIdStr = hexTimestamp.padLeft(8, '0') + '0000000000000000'
|
||||
return new ObjectId(objectIdStr)
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -97,7 +97,7 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], null);
|
||||
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
//测试通过匹配接口配房源效果
|
||||
|
||||
+33
-11
@@ -15,9 +15,10 @@ import groovy.sql.Sql
|
||||
|
||||
class CcbCommonUtils {
|
||||
|
||||
//
|
||||
// public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
|
||||
public static def citys_all = [ "上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
|
||||
public static def citys_fjw_all = ["深圳","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
|
||||
|
||||
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
|
||||
public static def citys_252 = ["潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州"]
|
||||
|
||||
@@ -69,6 +70,10 @@ class CcbCommonUtils {
|
||||
citys = CcbCommonUtils.citys_252
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}else if(countStr=="all"){
|
||||
citys = CcbCommonUtils.citys_fjw_all
|
||||
sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
}
|
||||
}else{
|
||||
citys = CcbCommonUtils.citys_all
|
||||
@@ -428,14 +433,14 @@ class CcbCommonUtils {
|
||||
return rows
|
||||
}
|
||||
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields){
|
||||
public static def findAllDistrictByCity(def city, def cat, def fields, def kind){
|
||||
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","汤堡小区")
|
||||
// q.put("name","鸿运大院")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
@@ -445,14 +450,13 @@ class CcbCommonUtils {
|
||||
keys.put(field, 1);
|
||||
}
|
||||
}
|
||||
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "[${city}] 原始:"+dbsour.count()
|
||||
def rets = []
|
||||
|
||||
def ids = []
|
||||
for(def d : dbsour){
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
if(kind){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
@@ -460,6 +464,16 @@ class CcbCommonUtils {
|
||||
ids << d._id.toString()
|
||||
rets << d
|
||||
}
|
||||
}else{
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
if(!ids.contains(d._id.toString())){
|
||||
ids << d._id.toString()
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println "[${city}] 小区量:"+rets.size()
|
||||
@@ -665,10 +679,6 @@ class CcbCommonUtils {
|
||||
}
|
||||
|
||||
public static def deleteGuidedPrice(def dealdate){
|
||||
def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
|
||||
|
||||
def sql2 = "delete from base.t0831_guided_price_gj_h where gujia_drc_dt='${dealdate}'"
|
||||
pgsqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
@@ -702,4 +712,16 @@ class CcbCommonUtils {
|
||||
return map
|
||||
}
|
||||
|
||||
public static def updateTrades(startDate, endDate, cDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
q.put("_id", new BasicDBObject('$gte', cDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
if(count>0){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("trade_date", endDate)
|
||||
DEAL_PRICE_COLL.updateMulti(q, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+10
-8
@@ -388,19 +388,21 @@ class DealPriceDataProcess {
|
||||
|
||||
def deleteThirdPartyData(){
|
||||
// ,"陵水","塔城","保亭","长沙","苏州","南昌","沧州","文昌","衡水"
|
||||
def citys = ["469005"]//CcbCommonUtils.citys_all
|
||||
// def citys = ["310000","440100","500000","410100","430100","330100","340100","320100","370200","320500","120000","442000","320300","350200","360100","320400","210200","530100","469005","469028","469029","230700","610700","533400","650400","511100","131100"]//CcbCommonUtils.citys_all
|
||||
def citys = ["上海","广州"]//CcbCommonUtils.citys_all
|
||||
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
|
||||
def rows = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def set = new HashSet()
|
||||
for(def cityid : citys) {
|
||||
for(def city : citys) {
|
||||
def cityid = cityCodeDatas.get(city)
|
||||
def rets = findPeGteThirdParty2(cityid, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
|
||||
println cityid +" "+rets.size()
|
||||
println city +" "+rets.size()
|
||||
if(rets){
|
||||
set.add(cityid)
|
||||
set.add(city)
|
||||
}
|
||||
for(def m : rets){
|
||||
// def cityId = cityCodeDatas.get(city)
|
||||
|
||||
// def sql = "delete from BIGDATA_THIRD_PARTY_PRICE_GJ where cityid='${cityId}' and commid='${m.commid}' and price_date=${month} and community_price=${m.price}"
|
||||
// sqlInstance236.executeUpdate(sql)
|
||||
def querySql = "select count(1) as count from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
@@ -409,7 +411,7 @@ class DealPriceDataProcess {
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println cityid +" "+m.commid+" (按价格)删除:"+total
|
||||
println city +" "+m.commid+" (按价格)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} and gujia_bds_comm_avprc=${m.price}"
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}else{
|
||||
@@ -419,7 +421,7 @@ class DealPriceDataProcess {
|
||||
total = r.count
|
||||
})
|
||||
if(total>0){
|
||||
println cityId +" "+m.commid+" (按小区)删除:"+total
|
||||
println city +" "+m.commid+" (按小区)删除:"+total
|
||||
def sql = "delete from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${m.commid}' and gujiabds_avprc_dt_val=${month} "
|
||||
pgsqlInstance.executeUpdate(sql)
|
||||
}
|
||||
@@ -450,7 +452,7 @@ class DealPriceDataProcess {
|
||||
}
|
||||
|
||||
def findPeGteThirdParty2(cityid, yearmonth){
|
||||
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>20"
|
||||
def sql = "SELECT gujia_comm_id,gujiabds_to_mkprc_mny from t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val=${yearmonth} and gujia_data_tp='第三方' and urbn_id='${cityid}' and gujia_mns_rto>25"
|
||||
def rets = []
|
||||
pgsqlInstance.eachRow(sql,{r->
|
||||
def map = [:]
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ class DistrictGuidePriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
|
||||
+2
-2
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28")
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-05-24", "2024-06-08")
|
||||
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
@@ -207,7 +207,7 @@ class GuiDingPriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], null);
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
|
||||
+6
@@ -2,6 +2,9 @@ package com.fangjia.script.ccbevaluation
|
||||
|
||||
import groovy.sql.Sql
|
||||
|
||||
/**
|
||||
* 建行使用
|
||||
*/
|
||||
class PgsqlCommUtils {
|
||||
|
||||
public static def database = "warehouse"
|
||||
@@ -19,6 +22,9 @@ class PgsqlCommUtils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public static def findAllDistrictCount(def cityid){
|
||||
def sql = "select count(gujia_comm_id) from base.t0831_bigdata_community_gj_h where urbn_id='${cityid}' and gujia_bds_bds_st=0";
|
||||
def count = 0
|
||||
|
||||
+1
-1
@@ -31,7 +31,7 @@ class StagingDistrictBatch extends BaseBatch{
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
for(def city : citys){
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"], null)
|
||||
def fenQiMap = getFenQiMap(dists);
|
||||
def rets = createCityData(dists, fenQiMap)
|
||||
rows.addAll(rets)
|
||||
|
||||
+70
-10
@@ -7,6 +7,7 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.looplineinfo.Point
|
||||
@@ -1288,10 +1289,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240607人工指导价.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['A']){
|
||||
if(d['J']){
|
||||
// def region = RegionData.formatRegion(d['B'], d['C'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
|
||||
@@ -1300,22 +1301,81 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// d['H'] = dist._id.toString()
|
||||
// }
|
||||
// }
|
||||
if(!d['I']){
|
||||
def dist = CommonDatautils.findDistrictById(d['C'])
|
||||
if(!d['A'] && d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
d['F'] = dist.region
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
|
||||
// d['H'] = dist.name
|
||||
d['G'] = dist.status
|
||||
// if(d['A']=="天门" || d['A']=="迪庆"){
|
||||
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
|
||||
// d['K'] = val.toInteger()
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
|
||||
//def filePath = "D:\\数据\\建行\\房产估值\\小区\\估家独家数据打标签.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
// if(d['B']){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrictById(d['B'])
|
||||
// if(obj){
|
||||
// def markMap = obj.mark_map
|
||||
// if(!obj.mark_map){
|
||||
// markMap = [:]
|
||||
// }
|
||||
// def data = new BasicDBObject()
|
||||
// markMap.put("exclusive", "是")
|
||||
// data.put("mark_map", markMap)
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['B'])), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// if(d['B'] && !d['D']){
|
||||
// def jsonObject = FangjiaClientUtils.newMatch(d['A'], d['B'], "property", "district")
|
||||
// if(jsonObject != null && jsonObject.credit>0) {
|
||||
// d['D'] = jsonObject.get("city")
|
||||
// d['E'] = jsonObject.get("district")
|
||||
// d['F'] = jsonObject.get("name")
|
||||
// def dbo = CommonDatautils.findDistrictById(jsonObject.get("id"))
|
||||
// if(dbo){
|
||||
// d['G'] = dbo.status
|
||||
// d['H'] = dbo.name_cat
|
||||
// d['I'] = dbo.estate_type2
|
||||
// d['J'] = "接口匹配"
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//// def objs = CommonDatautils.findDistrictList(d['A'],null, d['B'], "district")
|
||||
//// if(!objs){
|
||||
//// objs = CommonDatautils.findDistrictByAlias(d['A'], null, d['B'], "district")
|
||||
//// }
|
||||
//// if(objs && objs.size()==1){
|
||||
//// def o = objs[0]
|
||||
//// if(o){
|
||||
//// d['D'] = o.city
|
||||
//// d['E'] = o.region
|
||||
//// d['F'] = o.name
|
||||
//// d['G'] = o.status
|
||||
//// d['H'] = o.name_cat
|
||||
//// d['I'] = o.estate_type2
|
||||
//// d['J'] = "别名"
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//
|
||||
// }
|
||||
//})
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
|
||||
+2
-2
@@ -55,7 +55,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
// }
|
||||
// }
|
||||
//
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = ["上海","广州"]//CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
@@ -71,7 +71,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
def statMap = getDistrictStatByCity(city, weekend)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], null)
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
//def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)//老第三方均价表(现不用)
|
||||
|
||||
|
||||
+111
-40
@@ -133,26 +133,27 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def size = regionDatas.size()
|
||||
final CountDownLatch latch = new CountDownLatch(size)
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
for(def i = 0;i<regionDatas.size();i++){
|
||||
final def arr = regionDatas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try {
|
||||
totalCount.addAndGet(1)
|
||||
println totalCount.get() +" /${size} "+arr[0] + " "+ arr[1]
|
||||
def row = createDatas(arr[0], arr[1], startDate, endDate, cityCodeDataMap, arr[2])
|
||||
if(row){
|
||||
rets << row
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
// } catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
// };
|
||||
// })
|
||||
}
|
||||
latch.await()
|
||||
// latch.await()
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_DISTPRICE.dat"
|
||||
@@ -206,12 +207,14 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def allDistRegionPreMap = [:]
|
||||
def allLastMonthMap = [:]
|
||||
def allLastYearMap = [:]
|
||||
def allCcbPriceMap = [:]
|
||||
def allCcbPriceCaseCountMap = [:]
|
||||
for(def city : citys) {
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"]);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias"], null);
|
||||
allDistricts.addAll(districts)
|
||||
|
||||
def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
allPriceMap = allPriceMap + priceMap
|
||||
// def priceMap = CommonDatautils.queryDistrictStatByDate(city, startDate, endDate)
|
||||
// allPriceMap = allPriceMap + priceMap
|
||||
|
||||
def rentMap = CommonDatautils.findSqmetreRentByCityAndDate(city, endDate)
|
||||
allRentMap = allRentMap + rentMap
|
||||
@@ -220,9 +223,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def dealCountMap = findDealDatesByCity(cityCode, dealStartDate, startDate)
|
||||
allDealCountMap = allDealCountMap + dealCountMap
|
||||
|
||||
|
||||
def datas = findCommunityPriceByCity(cityCode, startDate.toString().substring(0,6))
|
||||
|
||||
def datasMap = findCommunityPriceByCityMap(cityCode, startDate.toString().substring(0,6))
|
||||
allCcbPriceMap = allCcbPriceMap + datasMap
|
||||
|
||||
|
||||
def caseCountMap = findCommunityPriceCountByCityMap(cityCode, startDate.toString().substring(0,6))
|
||||
allCcbPriceCaseCountMap = allCcbPriceCaseCountMap + caseCountMap
|
||||
|
||||
def cityRankMap = findCommunityCityRank(datas)
|
||||
allCityRankMap = allCityRankMap + cityRankMap
|
||||
|
||||
@@ -245,7 +254,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
allLastYearMap = allLastYearMap + lastYearMonthMap
|
||||
}
|
||||
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
def cityLists = districtRunningByCity(allDistricts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, allDealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
|
||||
if(cityLists){
|
||||
rets.addAll(cityLists)
|
||||
}
|
||||
@@ -263,7 +272,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
|
||||
|
||||
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def districtRunningByCity(districts, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
|
||||
def totalCount = new AtomicLong()
|
||||
def distCount = districts.size()
|
||||
final CountDownLatch latch = new CountDownLatch(distCount)
|
||||
@@ -276,7 +285,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap)
|
||||
def row = createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap,allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap)
|
||||
if(row){
|
||||
rets << row
|
||||
println totalCount.get() +"/"+distCount + dist.city+" "+dist.region+" "+dist.name+" "+row[4]+" "+row[6]
|
||||
@@ -323,37 +332,59 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap){
|
||||
def createDistDate(dist, startDate, endDate, cityCodeDataMap, allCcbPriceMap, allRentMap, dealCountMap, allCityRankMap, allRegionRankMap, allDistCityPreMap, allDistRegionPreMap, allLastMonthMap, allLastYearMap, allCcbPriceCaseCountMap){
|
||||
def sellDataType = 1
|
||||
def rentDataType = 1
|
||||
def sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
//房源查询是否按建行均价筛选房源
|
||||
def sellMap = null//findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
def sell_avg_price = null
|
||||
def sell_avg_area = null
|
||||
def sell_avg_total = null
|
||||
def sellStartDate = null
|
||||
def sell_count = 0
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<3)){
|
||||
sellDataType = 6
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
|
||||
|
||||
|
||||
if(allCcbPriceMap.containsKey(dist._id.toString())){
|
||||
sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
|
||||
// sell_avg_area = avgValue(sellMap.area)
|
||||
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
if(allCcbPriceCaseCountMap.containsKey(dist._id.toString())){
|
||||
sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
|
||||
}
|
||||
}
|
||||
if(sell_count){
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", startDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
if(!sellMap || (sellMap && sellMap.avg_price.size()<1)){
|
||||
sellDataType = 6
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -6), "yyyyMMdd").toInteger();
|
||||
sellMap = findHouses(dist.city, dist.region, "sell", sellStartDate, endDate, dist.name, dist.alias, dist.estate_type2)
|
||||
}
|
||||
}
|
||||
}
|
||||
if(sellMap){
|
||||
sell_count = sellMap.avg_price.size()
|
||||
if(sell_count>3){
|
||||
sell_avg_price = avgValue(sellMap.avg_price)
|
||||
if(sellMap){
|
||||
sell_avg_area = avgValue(sellMap.area)
|
||||
sell_avg_total = avgValue(sellMap.total_price)
|
||||
}else{
|
||||
sell_count = 0
|
||||
sellDataType = null
|
||||
sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
// if(sell_count>3){
|
||||
// if(allCcbPriceMap.containsKey(dist._id.toString())){
|
||||
// sell_avg_price = allCcbPriceMap.get(dist._id.toString())//avgValue(sellMap.avg_price)
|
||||
// sell_avg_area = avgValue(sellMap.area)
|
||||
// sell_avg_total = sell_avg_area*sell_avg_price//avgValue(sellMap.total_price)
|
||||
// sell_count = allCcbPriceCaseCountMap.get(dist._id.toString())
|
||||
// }else{
|
||||
// sell_count = 0
|
||||
// sellDataType = null
|
||||
// }
|
||||
// }else{
|
||||
// sell_count = 0
|
||||
// sellDataType = null
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def rent_avg_price = null
|
||||
def rent_avg_area = null
|
||||
def rent_avg_total = null
|
||||
@@ -381,9 +412,11 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
rentDataType = null
|
||||
}
|
||||
}
|
||||
if(!sell_avg_price){
|
||||
sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
}
|
||||
|
||||
//三个月,半年都没有数据时,用房价网价格,这块是否改为 建行价格
|
||||
// if(!sell_avg_price){
|
||||
// sell_avg_price = allPriceMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
// }
|
||||
if(!rent_avg_price){
|
||||
rent_avg_price = allRentMap.get(dist.city +"##"+ dist.region +"##"+ dist.name)
|
||||
}
|
||||
@@ -620,6 +653,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
def sell_avg_area = null
|
||||
def sell_avg_total = null
|
||||
def sellStartDate = null
|
||||
|
||||
def avgPrice = null
|
||||
if(region){
|
||||
avgPrice = findCcbRegionPriceByRegion(cityCodeDataMap.get(city), regionCodeDataMap.district_code,startDate.toString().substring(0,6))
|
||||
}else{
|
||||
avgPrice = findCcbCityPrice(cityCodeDataMap.get(city), startDate.toString().substring(0,6))
|
||||
}
|
||||
|
||||
|
||||
if (sell_count < 10) {
|
||||
sellDataType = 3
|
||||
sellStartDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -3), "yyyyMMdd").toInteger();
|
||||
@@ -633,7 +675,7 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
sellStartDate = startDate
|
||||
}
|
||||
if (sell_count) {
|
||||
sell_avg_price = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
|
||||
sell_avg_price = avgPrice//findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"avg_price")
|
||||
sell_avg_area = findHouseAvgByType(city, region, "sell", sellStartDate, endDate,"area")
|
||||
sell_avg_total = findHouseAvgByType(city, region, "sell", sellStartDate, endDate, "total_price")
|
||||
}
|
||||
@@ -1001,6 +1043,26 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return datas;
|
||||
}
|
||||
|
||||
def findCommunityPriceByCityMap(cityCode, month){
|
||||
def sql = "select gujia_admndiv_nm,gujia_bds_id,gujia_bds_clc_avprc from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def dataMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
dataMap.put(r.gujia_bds_id, r.gujia_bds_clc_avprc)
|
||||
})
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def findCommunityPriceCountByCityMap(cityCode, month){
|
||||
def sql = "SELECT gujia_comm_id,gujiabdsrt3mlgcasenum from t_app_p9pha_plda.avm_community_p_describe_gj_h WHERE urbn_id='${cityCode}' and gujia_bds_mo_val=${month}";
|
||||
def dataMap = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
if(r.gujiabdsrt3mlgcasenum){
|
||||
dataMap.put(r.gujia_comm_id, r.gujiabdsrt3mlgcasenum)
|
||||
}
|
||||
})
|
||||
return dataMap;
|
||||
}
|
||||
|
||||
def findCcbCityPrice(cityCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_city_price_gj_h where urbn_id='${cityCode}' and gujiabds_clc_yrmo_val=${month}";
|
||||
def avgPrice = null
|
||||
@@ -1021,6 +1083,15 @@ class BigdataCitypriceBatch extends BaseBatch{
|
||||
return regionPriceMap;
|
||||
}
|
||||
|
||||
def findCcbRegionPriceByRegion(cityCode, regionCode, month){
|
||||
def sql = "SELECT gujia_bds_clc_avprc from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${month} and gujia_admndiv_id='${regionCode}' and urbn_id='${cityCode}'";
|
||||
def price = null
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
price = r.gujia_bds_clc_avprc
|
||||
})
|
||||
return price;
|
||||
}
|
||||
|
||||
|
||||
def districtCityPrictRate(cityPrice, datas){
|
||||
def distCityRateMap = [:]
|
||||
|
||||
+7
-7
@@ -159,7 +159,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_FJW.dat"
|
||||
}
|
||||
@@ -246,7 +246,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDSTRU_FJW.dat"
|
||||
}
|
||||
@@ -304,7 +304,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSETYPE_FJW.dat"
|
||||
}
|
||||
@@ -362,7 +362,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMBUILDYEAR_FJW.dat"
|
||||
}
|
||||
@@ -425,7 +425,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMFEE_FJW.dat"
|
||||
}
|
||||
@@ -504,7 +504,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMHOUSEUSE_FJW.dat"
|
||||
}
|
||||
@@ -590,7 +590,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch {
|
||||
rets << ret
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10";
|
||||
def fileName = "/ccb/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
|
||||
def fileName = "/ccb/${curDate}/MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_POSITION_FJW.dat"
|
||||
}
|
||||
|
||||
+14
-15
@@ -4,17 +4,15 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
|
||||
import com.fangjia.script.ccbevaluation.CcbFtpUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.jcraft.jsch.ChannelSftp
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.sql.ResultSetMetaData
|
||||
|
||||
/**
|
||||
* 建行融合数据导出
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
@@ -239,7 +237,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'";
|
||||
def sql = "select urbn_id,gujia_bld_tp_val,gujia_bld_tp_val_coef,gujia_clc_yrmo_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_city_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}'";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
@@ -255,23 +253,23 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
|
||||
println "EVALUATE_RATE_CITY_FJW_ADD 数据文件输出完成......"
|
||||
println "EVALUATE_RATE_CITY_FJW_HIS_ADD 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "EVALUATE_RATE_CITY_FJW_ADD 上传Ftp完成......"
|
||||
println "EVALUATE_RATE_CITY_FJW_HIS_ADD 上传Ftp完成......"
|
||||
}
|
||||
|
||||
def exportEvaluateRateComm(queryDate){
|
||||
def allRows = []
|
||||
for(def city : citys) {
|
||||
def cityCode = cityCodeMap.get(city)
|
||||
def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,data_crt_tm,data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' ";
|
||||
def sql = "select urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_clc_yrmo_val,basic_price,area_avg,unitprc_avg,comm_type,area_avg_coef,unitprc_avg_coef,basic_price_coef,buildarea_coef,type_coef,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_evaluate_rate_comm_gj_h where urbn_id='${cityCode}' and gujia_clc_yrmo_val = '${queryDate}' ";
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
@@ -284,6 +282,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
data << r.unitprc_avg
|
||||
data << r.comm_type
|
||||
data << r.area_avg_coef
|
||||
data << r.unitprc_avg_coef
|
||||
data << r.basic_price_coef
|
||||
data << r.buildarea_coef
|
||||
data << r.type_coef
|
||||
@@ -296,16 +295,16 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_${curDate}_0001.dat"
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
|
||||
|
||||
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 数据文件输出完成......"
|
||||
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 数据文件输出完成......"
|
||||
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
|
||||
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_ADD_ 上传Ftp完成......"
|
||||
println "FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_ 上传Ftp完成......"
|
||||
}
|
||||
|
||||
|
||||
|
||||
+255
-3
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.script.commData
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
@@ -8,11 +9,18 @@ import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.client.MongoCursor
|
||||
import com.mongodb.client.model.Accumulators
|
||||
import com.mongodb.client.model.Aggregates
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.json.JsonSlurper
|
||||
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
@@ -110,6 +118,8 @@ class CommonDatautils {
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static final DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009")
|
||||
public static final DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
public static DSCollection getHouseDataStoreConnection(){
|
||||
if(house_dataStoreConnection != null) {
|
||||
@@ -267,7 +277,7 @@ class CommonDatautils {
|
||||
q.put("status", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name","揭흩소蹈")
|
||||
// q.put("name","托斯卡纳庄园")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -276,7 +286,7 @@ class CommonDatautils {
|
||||
}
|
||||
}
|
||||
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
|
||||
def cursor = DbUtils.seaweed.find(q, keys).batchSize(5000)
|
||||
List<DBObject> rets = []
|
||||
for(def obj : cursor){
|
||||
rets << obj
|
||||
@@ -1459,6 +1469,28 @@ class CommonDatautils {
|
||||
return obj;
|
||||
}
|
||||
|
||||
static def findSeaweedListByMappingAlias(def city, def region, def name, def cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
if(region){
|
||||
q.put('region', region)
|
||||
}
|
||||
|
||||
q.put('city',city)
|
||||
q.put('cat', cat)
|
||||
q.put('status',0)
|
||||
|
||||
BasicDBObject query = new BasicDBObject();
|
||||
query.putAt(QueryOperators.IN, [name]);
|
||||
q.put("mapping_alias", query);
|
||||
|
||||
def dbcursor = SEAWEED_COLL.find(q);
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
static def findSeaweeds(city, cat, name_cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put('city',city)
|
||||
@@ -1691,7 +1723,34 @@ class CommonDatautils {
|
||||
}
|
||||
return regionMap;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def getAllCityFullRegions(){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("category","region");
|
||||
//queryCondition.put("show",1);
|
||||
def dbCursor = REGION_BLOCK_COLL.find(queryCondition)
|
||||
def regionMap = [:];
|
||||
for(DBObject obj : dbCursor){
|
||||
def city = obj.get("city");
|
||||
def region = obj.get("name");
|
||||
def pattern = obj.get("pattern");
|
||||
if(region.indexOf("周边")==-1 && region.indexOf("其他")==-1) {
|
||||
def r_map = regionMap.get(city);
|
||||
if(!r_map){
|
||||
r_map = [:]
|
||||
regionMap.put(city, r_map);
|
||||
}
|
||||
|
||||
if(!pattern){
|
||||
pattern = [];
|
||||
pattern << region;
|
||||
}
|
||||
r_map.put(region, ["pattern":pattern, "full_region":obj.get("org_name")]);
|
||||
}
|
||||
}
|
||||
return regionMap;
|
||||
}
|
||||
|
||||
|
||||
public static def getRegionsByCity(def city){
|
||||
@@ -1834,6 +1893,25 @@ class CommonDatautils {
|
||||
def dbo = null;
|
||||
while(cur.hasNext()) {
|
||||
dbo = cur.next()
|
||||
if(dbo.value_type == 1){
|
||||
dbo.cal_type = "指导价"
|
||||
}else if(dbo.value_type == 2){
|
||||
dbo.cal_type = "自身样本"
|
||||
}else if(dbo.value_type == 3){
|
||||
dbo.cal_type = "第三方"
|
||||
}else if(dbo.value_type == 4){
|
||||
dbo.cal_type = "交易数据"
|
||||
}else if(dbo.value_type == 5){
|
||||
dbo.cal_type = "周边楼盘"
|
||||
}else if(dbo.value_type == 6){
|
||||
dbo.cal_type = "推导"
|
||||
}else if(dbo.value_type == 9){
|
||||
dbo.cal_type = "神经网络"
|
||||
}else if(dbo.value_type == 10){
|
||||
dbo.cal_type = "无方式"
|
||||
}else{
|
||||
dbo.cal_type = ""
|
||||
}
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
@@ -4496,6 +4574,28 @@ class CommonDatautils {
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def findAllTownRanges(){
|
||||
def q = new BasicDBObject()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("region", 1)
|
||||
keys.put("town", 1)
|
||||
def dbcursor = TOWN_POLYLINECOLL.find(q, keys)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def lists = townMap.get(d.city)
|
||||
if(!lists){
|
||||
lists = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.region
|
||||
t.town = d.town
|
||||
lists << t
|
||||
townMap.put(d.city, lists)
|
||||
}
|
||||
return townMap
|
||||
}
|
||||
|
||||
static def queryFullNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = administrative_division_100_CONN.find(q)
|
||||
@@ -4530,6 +4630,108 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def queryFullNameByCity(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","城市")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def cityMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def city = RegionData.formatCity(d.city)
|
||||
if(city){
|
||||
cityMap.put(city, d)
|
||||
}else{
|
||||
//println d.city +" 格式化不出"
|
||||
}
|
||||
|
||||
}
|
||||
return cityMap
|
||||
}
|
||||
|
||||
static def queryDistNameByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.mzb_district_name){
|
||||
def regions = regionMap.get(d.city)
|
||||
if(!regions){
|
||||
regions = []
|
||||
}
|
||||
regions << d
|
||||
regionMap.put(d.city, regions)
|
||||
}else{
|
||||
println d.province+" "+d.district +" 格式化不出民政部区域"
|
||||
}
|
||||
}
|
||||
return regionMap
|
||||
}
|
||||
|
||||
static def queryCommunityByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("province",new BasicDBObject('$in', ['福建省','重庆市']))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"#"+d.district
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
return townMap
|
||||
|
||||
}
|
||||
|
||||
|
||||
static def queryNewTownByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
def t = [:]
|
||||
t.region = d.district
|
||||
t.town = d.street
|
||||
list << t
|
||||
townMap.put(key, list)
|
||||
}
|
||||
return townMap
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryDistNameCity(){
|
||||
def q = new BasicDBObject()
|
||||
@@ -4600,5 +4802,55 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
static def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
package com.fangjia.script.fuzhou
|
||||
|
||||
|
||||
+6
-1
@@ -283,7 +283,12 @@ class DistrictSellStatBatch extends BaseBatch {
|
||||
def dbcursor = DISTRICT_SELL_STAT.find(q)
|
||||
def priceMap = [:]
|
||||
for(def d : dbcursor){
|
||||
priceMap.put(d.region +"##"+d.district_name, d)
|
||||
if(d.estate_type=="别墅"){
|
||||
priceMap.put(d.region +"##"+d.district_name+"##别墅", d)
|
||||
}else{
|
||||
priceMap.put(d.region +"##"+d.district_name+"##小区", d)
|
||||
}
|
||||
|
||||
}
|
||||
return priceMap
|
||||
}
|
||||
|
||||
+4
-4
@@ -29,11 +29,11 @@ class FzDataResultStat {
|
||||
|
||||
static main(args) {
|
||||
def task = new FzDataResultStat()
|
||||
// task.priceMarkupRun();
|
||||
task.priceMarkupRun();
|
||||
// task.regionPriceTrendRun()
|
||||
//// task.margeExcel()
|
||||
// task.saveCsvData()
|
||||
task.testNewPrice()
|
||||
// task.testNewPrice()
|
||||
}
|
||||
|
||||
def margeExcel(){
|
||||
@@ -58,7 +58,7 @@ class FzDataResultStat {
|
||||
}
|
||||
def priceMarkupRun(){
|
||||
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
// List<String> list1= FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_${curMonth}.csv"))
|
||||
// def dists = CommonDatautils.findAllDistricts("¸£ÖÝ", "district", ["status"])
|
||||
@@ -82,7 +82,7 @@ class FzDataResultStat {
|
||||
}
|
||||
|
||||
// List<String> list2 = FileUtils.readLines(new File("D:\\Êý¾Ý\\¸£ÖÝ\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
|
||||
def list2 = findFzDistrictStat(lastMonth)
|
||||
def list2 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")))
|
||||
def map2 = [:]
|
||||
for(def obj : list2) {
|
||||
def key = obj.city + "##" + obj.region + "##" + obj.name
|
||||
|
||||
+2
-2
@@ -22,8 +22,8 @@ class FzDistrictMatch {
|
||||
// task.addPrice()
|
||||
// task.addThirdPrice()
|
||||
|
||||
// task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
|
||||
task.longyanPrice()
|
||||
task.addThirdPrice2("D:\\数据\\福州\\价格比对_20240315.xlsx")
|
||||
// task.longyanPrice()
|
||||
}
|
||||
|
||||
def longyanPrice(){
|
||||
|
||||
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
|
||||
//福州第三方均价统计,发送钉钉消息
|
||||
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
|
||||
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202402.csv")
|
||||
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202403.csv")
|
||||
def count0 = 0
|
||||
def count1 = 0
|
||||
def count2 = 0
|
||||
|
||||
@@ -238,6 +238,7 @@ class FiledStrand {
|
||||
}
|
||||
|
||||
static def getValue(def val){
|
||||
val = val.replace(",","")
|
||||
val = getNumber(val);
|
||||
if(val){
|
||||
if(val.indexOf(".")>-1 && !val.toString().endsWith(".0")){
|
||||
|
||||
+697
-4
@@ -1,23 +1,115 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class DevProcess {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevProcess();
|
||||
task.run()
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
|
||||
// task.calAddrDistince()
|
||||
// task.updateCityAndDist()
|
||||
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
|
||||
// task.addDevAddres2()//添加相差2公里范围内的地址
|
||||
// task.matchDevAndAddAddress()
|
||||
|
||||
// task.stat()
|
||||
// task.addNewDev()
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo43()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
// task.addDevShape()//添加开发区围栏
|
||||
// task.matchDevTownship()//添加乡镇街道
|
||||
|
||||
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
|
||||
}
|
||||
|
||||
|
||||
|
||||
def parseCityAndDist(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\等级开发区.xlsx",{d->
|
||||
if(!d['J']){
|
||||
// def obj = getMapInfo(null, d['A']+d['C'])
|
||||
// if(obj){
|
||||
// d['E'] = obj.city.toString()
|
||||
// d['F'] = obj.district.toString()
|
||||
// d['G'] = obj.formatted_address.toString()
|
||||
// d['H'] = obj.province.toString()
|
||||
// }
|
||||
|
||||
def obj = FangjiaClientUtils.formatNinelevel(null, d['C']);
|
||||
if(obj) {
|
||||
def jsonObject = obj.get("result")
|
||||
|
||||
if (jsonObject) {
|
||||
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
|
||||
d['J'] = jsonObject.get("province")
|
||||
}
|
||||
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
|
||||
d['K'] = jsonObject.get("city")
|
||||
}
|
||||
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
|
||||
d['L'] = jsonObject.get("district")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def getMapInfo(city, address){
|
||||
def json = GaoDeApiUtils.geo(city, address)
|
||||
println json
|
||||
if(json && json.status=="1"){
|
||||
def ret = json.geocodes
|
||||
if(ret){
|
||||
def obj = ret.get(0)
|
||||
def str = obj.location
|
||||
def level = obj.level
|
||||
|
||||
// if(str && ["门牌号","单元号","道路","道路交叉路口","热点商圈","村庄","兴趣点","门址","住宅区"].contains(level)){
|
||||
def arrs = str.split(",")
|
||||
return ["lng":arrs[0], "lat":arrs[1],"province":obj.province,"city":obj.city, "district":obj.district, "formatted_address":obj.formatted_address]
|
||||
// }else{
|
||||
// println address +" "+level
|
||||
// return null
|
||||
// }
|
||||
}
|
||||
}
|
||||
}
|
||||
def run(){
|
||||
def rets = ExcelUtils.read("D:\\数据\\开发区\\附件全国农村创业园区(基地)目录2021.xls")
|
||||
@@ -77,10 +169,611 @@ class DevProcess {
|
||||
}else{
|
||||
println udata.name +" 已存在"
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def findDev(city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
q.put("city", city)
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def devSplitRoad(){
|
||||
def i = 0;
|
||||
while(i<20){
|
||||
def fileName = "/home/goojia/gj/qzd-${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\福建省.xlsx"
|
||||
}
|
||||
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['E'] && !d['G']){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("dev") && jsonObject1.get("dev") != "null") {
|
||||
def devName = jsonObject1.get("dev")
|
||||
def obj = findDev(d['B'], devName)
|
||||
d['F'] = devName
|
||||
if (obj) {
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(d['C'] && !d['S']) {
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['C'])
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if (jsonObject.get("province") && jsonObject.get("province") != "null") {
|
||||
d['G'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if (jsonObject.get("city") && jsonObject.get("city") != "null") {
|
||||
d['H'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if (jsonObject.get("district") && jsonObject.get("district") != "null") {
|
||||
d['I'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if (jsonObject.get("township") && jsonObject.get("township") != "null") {
|
||||
d['J'] = jsonObject.get("township")
|
||||
}
|
||||
|
||||
if (jsonObject && street && streetNumber && street != "null") {
|
||||
d['K'] = jsonObject.get("street")
|
||||
def road = jsonObject.get("street")
|
||||
if (streetNumber != 'null') {
|
||||
d['L'] = jsonObject.get("streetNumber")
|
||||
road = road + jsonObject.get("streetNumber")
|
||||
} else {
|
||||
d['N'] = "信息不完整"
|
||||
}
|
||||
|
||||
if (!d['N'] && !d['C'].contains(road)) {
|
||||
d['N'] = "原地址不含路"
|
||||
}
|
||||
|
||||
if (!d['N']) {
|
||||
def flag = road.matches('^[\u4e00-\u9fa5]+?(路|大道|巷|胡同|街).+[弄号]$')
|
||||
if (!flag) {
|
||||
d['N'] = "路非法"
|
||||
}
|
||||
def roadsize = jsonObject.get("street").size()
|
||||
if (!d['N'] && (roadsize < 3 || roadsize > 7)) {
|
||||
d['N'] = "路太长|路太短"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d['N'] = "无地址"
|
||||
}
|
||||
|
||||
if (jsonObject.get("dev") && jsonObject.get("dev") != "null") {
|
||||
def devName = jsonObject.get("dev")
|
||||
def obj = findDev(d['B'], devName)
|
||||
d['M'] = devName
|
||||
if (obj) {
|
||||
d['S'] = obj._id.toString()
|
||||
d['T'] = obj.name
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def devNameSplitRoad(){
|
||||
def fileName = "/home/goojia/gj/company.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\开发区\\56万需修改地址.xlsx"
|
||||
}
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['P'] && d['P']=="路正常" && !d['O']){
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['E'])
|
||||
if(jsonObject){
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
|
||||
d['J'] = jsonObject.get("province")
|
||||
}
|
||||
|
||||
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
|
||||
d['K'] = jsonObject.get("city")
|
||||
}
|
||||
|
||||
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
|
||||
d['L'] = jsonObject.get("district")
|
||||
}
|
||||
|
||||
if(jsonObject && street && streetNumber && street!="null"){
|
||||
d['M'] = jsonObject.get("street")
|
||||
if(streetNumber!='null'){
|
||||
d['N'] = jsonObject.get("streetNumber")
|
||||
}
|
||||
}
|
||||
|
||||
if(jsonObject.get("dev") && jsonObject.get("dev")!="null"){
|
||||
d['O'] = jsonObject.get("dev")
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
def addFlag(){
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Downloads\\企业园区地址.xlsx",{d->
|
||||
if(d['H'] && d['D']){
|
||||
if(d['D'].contains(d['H'])){
|
||||
d['O'] = "是"
|
||||
}else{
|
||||
def newName = null
|
||||
if(d['H'].toString().startsWith(d['I'])){
|
||||
newName = d['H'].toString().substring(d['I'].toString().length())
|
||||
}else if(d['H'].toString().startsWith(d['I']+"省")){
|
||||
newName = d['H'].toString().substring(d['I'].toString().length()+1)
|
||||
}
|
||||
if(newName && d['D'].contains(newName)){
|
||||
d['O'] = "是2"
|
||||
}
|
||||
}
|
||||
}
|
||||
},1);
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
|
||||
if(!d['Q'] && d['L']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['L'])
|
||||
q.put("city",d['G'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def calAddrDistince(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
|
||||
if(d['M'] && d['N'] && d['Y'] && d['Z']){
|
||||
def addr1 = getFullAddress(d['J'], d['K'],d['L'],d['M'] , d['N'])
|
||||
def addr2 = getFullAddress(d['V'], d['W'], d['X'], d['Y'] , d['Z'])
|
||||
|
||||
def json1 = GaoDeApiUtils.geo(d['K'], addr1)
|
||||
def json2 = GaoDeApiUtils.geo(d['W'], addr2)
|
||||
def arrs1 = getLocation(json1)
|
||||
def arrs2 = getLocation(json2)
|
||||
if(arrs1 && arrs2){
|
||||
def distance = GeoUtils.calculateDistance(arrs1[0].toString().toDouble(), arrs1[1].toString().toDouble(), arrs2[0].toString().toDouble() , arrs2[1].toString().toDouble() )
|
||||
d['AA'] = distance
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
def getFullAddress(province, city, region, street, streetName){
|
||||
def addr = ""
|
||||
if(addr){
|
||||
addr = province
|
||||
}
|
||||
if(city){
|
||||
addr = addr + city
|
||||
}
|
||||
if(region){
|
||||
addr = addr + region
|
||||
}
|
||||
addr = addr + street + streetName
|
||||
return addr
|
||||
}
|
||||
def getLocation(json1){
|
||||
def arrs1 = null
|
||||
if(json1 && json1.status=="1") {
|
||||
def ret = json1.geocodes
|
||||
if (ret) {
|
||||
def str = ret.get(0).location
|
||||
if(str){
|
||||
arrs1 = str.split(",")
|
||||
}
|
||||
}
|
||||
}
|
||||
return arrs1
|
||||
}
|
||||
|
||||
def updateCityAndDist(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\等级开发区 - 汇总.xlsx",{d->
|
||||
if(d['L']){
|
||||
def udata = new BasicDBObject()
|
||||
if(d['L']=="采用拆分结果"){
|
||||
if(d["I"]){
|
||||
udata.put("province", d["I"])
|
||||
}
|
||||
if(d["J"]){
|
||||
udata.put("city", d["J"])
|
||||
}
|
||||
if(d["K"]){
|
||||
udata.put("district", d["K"])
|
||||
}
|
||||
}else if(d['L']=="采用地图结果"){
|
||||
if(d["H"]){
|
||||
udata.put("province", d["H"])
|
||||
}
|
||||
if(d["E"]){
|
||||
udata.put("city", d["E"])
|
||||
}
|
||||
if(d["F"]){
|
||||
udata.put("district", d["F"])
|
||||
}
|
||||
}
|
||||
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['C']))
|
||||
if(obj && (!obj.city && !obj.district)){
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addDevAddres1(){
|
||||
def rets = []
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\企业园区地址-新.xlsx",{d->
|
||||
if(d['O'] && d['P']=="正常" && d['U']){
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
|
||||
if(obj){
|
||||
d['AB'] = "存在"
|
||||
def addr_alias = obj.addr_alias
|
||||
if(!addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
def old_count = addr_alias.size()
|
||||
if(!addr_alias.contains(d['M']+d['N'])){
|
||||
addr_alias << d['M']+d['N']
|
||||
}
|
||||
if(old_count != addr_alias.size()){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("addr_alias", addr_alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}else{
|
||||
rets << d
|
||||
println "匹配:"+d['U']+" 原始:"+d['H']
|
||||
}
|
||||
}
|
||||
},0)
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
def addDevAddres2(){
|
||||
|
||||
def rets = []
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d->
|
||||
if((d['R'] && d['R'].toString()=="正常") || (d['P'] && d['P'].toString()=="正常")){
|
||||
def obj = null
|
||||
if(d['U']){
|
||||
try{
|
||||
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
|
||||
}catch(Exception e){
|
||||
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['U']))
|
||||
}
|
||||
}
|
||||
|
||||
if(!obj){
|
||||
try{
|
||||
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H']))
|
||||
}catch(Exception e){
|
||||
obj = development_zone_CONN.findOne(new BasicDBObject("name", d['H']))
|
||||
}
|
||||
}
|
||||
if(obj){
|
||||
d['AB'] = "存在"
|
||||
def addr_alias = obj.addr_alias
|
||||
if(!addr_alias){
|
||||
addr_alias = []
|
||||
}
|
||||
def old_count = addr_alias.size()
|
||||
|
||||
if(d['R'] && d['R']=="正常" && !addr_alias.contains(d['Y']+d['Z']) && d['X'] && d['F'] && d['X']==d['F']){
|
||||
addr_alias << d['Y']+d['Z']
|
||||
}
|
||||
|
||||
if(d['P'] && d['P']=='正常' && !addr_alias.contains(d['M']+d['N']) && d['F'] && d['L'] && d['L']==d['F']){
|
||||
addr_alias << d['M']+d['N']
|
||||
}
|
||||
|
||||
if(old_count != addr_alias.size()){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("addr_alias", addr_alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}else{
|
||||
// rets << d
|
||||
println "匹配:"+d['U']+" 原始:"+d['H']
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
def total = 0
|
||||
for(def d : dbcursor){
|
||||
if(d.addr_alias){
|
||||
total = total + d.addr_alias.size()
|
||||
println d.addr_alias.size()
|
||||
}
|
||||
}
|
||||
println total
|
||||
}
|
||||
|
||||
def addNewDev(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\待新增国家级以上开发区.xlsx",{d->
|
||||
def udata = new BasicDBObject()
|
||||
udata.address = d['M']+d['N']
|
||||
udata.create_time = new Date()
|
||||
if(d['G']){
|
||||
if(d['G'].contains("国家级")){
|
||||
udata.level = "国家级"
|
||||
}else if(d['G'].contains("省级")){
|
||||
udata.level = "省级"
|
||||
}
|
||||
}
|
||||
|
||||
udata.name = d['H']
|
||||
udata.province = d['J']
|
||||
udata.city = d['K']
|
||||
udata.district = d['L']
|
||||
udata.type = d['G']
|
||||
|
||||
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
|
||||
if(!obj){
|
||||
development_zone_CONN.save(udata)
|
||||
}else{
|
||||
println udata.name +" 已存在"
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addNewDev2(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\河北省官网发布190开发区.xlsx",{d->
|
||||
if(d['F'] && d['F']=="新增"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.create_time = new Date()
|
||||
udata.province = d['G']
|
||||
udata.city = d['I']
|
||||
udata.district = d['J']
|
||||
udata.level = "省级"
|
||||
udata.name = d['D']
|
||||
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", udata.name))
|
||||
if(!obj){
|
||||
development_zone_CONN.save(udata)
|
||||
}else{
|
||||
println udata.name +" 已存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def createDevMapping(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
for(def d : dbcursor){
|
||||
def formatAddress = null
|
||||
if(d.address){
|
||||
formatAddress = AddressUtil.formatAddress(d.city, d.address)
|
||||
}
|
||||
if(formatAddress && !formatAddress.endsWith("号") && !formatAddress.endsWith("弄")){
|
||||
formatAddress = null
|
||||
}
|
||||
def addr_alias = []
|
||||
if(formatAddress){
|
||||
addr_alias << formatAddress
|
||||
}
|
||||
if(d.addr_alias){
|
||||
addr_alias = addr_alias + d.addr_alias
|
||||
addr_alias.unique()
|
||||
}
|
||||
if(addr_alias){
|
||||
addr_alias.each{addr->
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", d.province)
|
||||
udata.put("city", d.city)
|
||||
udata.put("district", d.district)
|
||||
udata.put("name",d.name)
|
||||
udata.put("type", "地址")
|
||||
if(d.township){
|
||||
udata.put("street", d.township)
|
||||
}
|
||||
udata.put("address", addr)
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def deleteAddr(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\删除开发区地址.xlsx",{d->
|
||||
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", new ObjectId(d['A'])))
|
||||
});
|
||||
}
|
||||
|
||||
def deleteAddr2(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\56万中待删除地址.xlsx",{d->
|
||||
if(d['Q'] && d['M']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", new ObjectId(d['Q']))
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def q2 = new BasicDBObject()
|
||||
q2.put("name", obj.name)
|
||||
q2.put("address", d['I']+d['J'])
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q2)
|
||||
if(sameObj){
|
||||
DEV_MAPPING_CONN.remove(new BasicDBObject("_id", sameObj._id))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
def synchDevTo43(){
|
||||
def dbcursor = development_zone_CONN.find()
|
||||
def rets = []
|
||||
for(def r : dbcursor){
|
||||
rets << r
|
||||
}
|
||||
DEVELOPMENT_ZONE_43_COLL.insert(rets)
|
||||
}
|
||||
|
||||
|
||||
def matchDevAndAddAddress(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\浙江-湖州路正常(2.6+15万修正).xlsx",{d->
|
||||
if(d['R']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['R'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
|
||||
d['P'] = obj._id.toString()
|
||||
d['Q'] = obj.name
|
||||
def address = d['L']+d['M']
|
||||
q.put("address", address)
|
||||
// def street = d['G']
|
||||
// q.put("township", street)
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q)
|
||||
if(!sameObj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", obj.province)
|
||||
if(obj.city){
|
||||
udata.put("city", obj.city)
|
||||
}
|
||||
if(obj.district){
|
||||
udata.put("district", obj.district)
|
||||
}
|
||||
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "地址")
|
||||
udata.put("address",address)
|
||||
// udata.put("type", "乡镇街道")
|
||||
// udata.put("township", street)
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
println obj.name +" "+address
|
||||
}
|
||||
}
|
||||
}
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matchDevTownship(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
|
||||
if(d['C']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['C'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['E'] = obj._id.toString()
|
||||
def street = d['D']
|
||||
q.put("township", street)
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q)
|
||||
if(!sameObj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", obj.province)
|
||||
if(obj.city){
|
||||
udata.put("city", obj.city)
|
||||
}
|
||||
if(obj.district){
|
||||
udata.put("district", obj.district)
|
||||
}
|
||||
|
||||
udata.put("name",obj.name)
|
||||
// udata.put("type", "乡镇街道")
|
||||
// udata.put("township", street)
|
||||
|
||||
udata.put("type", "区域")
|
||||
udata.put("dev_district", street)
|
||||
udata.put("c_date",new Date())
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
println obj.name +" "+street
|
||||
}
|
||||
}
|
||||
}
|
||||
},1);
|
||||
}
|
||||
|
||||
def addAlias(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区别名.xlsx",{d->
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['A'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['D'] = obj._id.toString()
|
||||
def arrs = d['B'].split("、")
|
||||
def alias = obj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
for(def s : arrs){
|
||||
if(!alias.contains(s)){
|
||||
alias << s
|
||||
}
|
||||
}
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addDevShape(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\开发区围栏.xlsx",{d->
|
||||
if(d['W']){
|
||||
def s = d['S']
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
Map<String, Object> map = DataHandleUtils.JsonToMap(jsonObject);
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", map)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",new ObjectId(d['L'])), new BasicDBObject('$set',udata))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addDevStreet(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\区域匹配开发区.xlsx",{d->
|
||||
if(d['J']=="是"){
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("district", d['D']).append("type","乡镇街道"))
|
||||
for(def obj : dbcursor){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province" , obj.province)
|
||||
o.put("city" , obj.city)
|
||||
o.put("district" , obj.mzb_district_name)
|
||||
o.put("name" , d['E'])
|
||||
o.put("type" , "乡镇街道")
|
||||
o.put("township" , obj.street)
|
||||
DEV_MAPPING_CONN.save(o)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
class DistrictAliasProcess {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictAliasProcess();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\小区\\名称在映射别名中-第二批.xlsx",{ d->
|
||||
if(d['H']=="是"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("status", 1)
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(d['A'])), new BasicDBObject('$set', udata))
|
||||
}else if(d['H']=="否"){
|
||||
def aliasDist = d['F']
|
||||
def arrs = aliasDist.split("-")
|
||||
def obj = CommonDatautils.findDistrict(d['B'], arrs[0], arrs[1],"district")
|
||||
if(obj){
|
||||
def alias = obj.alias
|
||||
def mapping_alias = obj.mapping_alias
|
||||
def udata = new BasicDBObject()
|
||||
if(alias && alias.contains(d['D'])){
|
||||
alias.remove(d['D'])
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
if(mapping_alias && mapping_alias.contains(d['D'])){
|
||||
mapping_alias.remove(d['D'])
|
||||
udata.put("mapping_alias", mapping_alias)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata))
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+53
-25
@@ -6,6 +6,7 @@ import com.fangjia.script.ccbevaluation.PgsqlCommUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
@@ -32,7 +33,9 @@ class GXDdistrictMatch {
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
task.restoreDist()//恢复小区status=0
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
task.runMatch()
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
@@ -133,6 +136,22 @@ class GXDdistrictMatch {
|
||||
})
|
||||
}
|
||||
|
||||
def runMatch(){
|
||||
def fileName = "D:\\数据\\福州\\福州贝壳数据.xlsx"//11城重新匹配.xlsx
|
||||
def city = "福州"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
println d['C']
|
||||
if(d['B'] && d['A']){
|
||||
def region = RegionData.formatRegion(city, d['B'])
|
||||
if (region) {
|
||||
d = match(city, region, d)
|
||||
} else {
|
||||
println d['A'] + "#" + d['E'] + " 区域无法格式化"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateDistStatus(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城(全).xlsx"
|
||||
def index = 0
|
||||
@@ -187,27 +206,32 @@ class GXDdistrictMatch {
|
||||
}
|
||||
|
||||
def saveToDB(){
|
||||
def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
// def fileName = "D:\\数据\\国信达\\国信达11城未匹配数据.xlsx"
|
||||
def fileName = "D:\\数据\\数据建设\\小区\\东莞未匹配成交数据.xlsx"
|
||||
ExcelUtils.modify(fileName, { d ->
|
||||
if(d['U'] && d['AG']!='过滤'){
|
||||
if(d['A']){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['AD'])
|
||||
udata.put("region", d['AE'])
|
||||
def name = d['X'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
def name = d['C'].replaceAll('[·\\(\\)()]|·', '').replace("建设中", "")
|
||||
udata.put("name", name)
|
||||
|
||||
def alias = []
|
||||
if(name!=d['C']){
|
||||
alias << d['C']
|
||||
}
|
||||
if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||
alias << d['AI']
|
||||
if(d['G'] && name!=d['G']){
|
||||
alias << d['G']
|
||||
}
|
||||
// if(d['AI'] && !alias.contains(d['AI']) && d['AI']!=name){
|
||||
// alias << d['AI']
|
||||
// }
|
||||
if(alias){
|
||||
udata.put("alias", alias)
|
||||
}
|
||||
|
||||
udata.put("address", d['AF'])
|
||||
def addr = AddressUtil.formatAddress(d['A'], d['D'])
|
||||
if(addr){
|
||||
udata.put("address", addr)
|
||||
}
|
||||
|
||||
udata.put("status", "0")
|
||||
udata.put("name_cat", "0")
|
||||
udata.put("cat", "district")
|
||||
@@ -215,11 +239,11 @@ class GXDdistrictMatch {
|
||||
|
||||
def estateType2 = "普通住宅"
|
||||
def type = ""
|
||||
if(d['AH']){
|
||||
type = d['AH']
|
||||
}else{
|
||||
type = d['Y']
|
||||
}
|
||||
// if(d['AH']){
|
||||
// type = d['AH']
|
||||
// }else{
|
||||
// type = d['Y']
|
||||
// }
|
||||
if(type){
|
||||
if(type.contains("别墅")){
|
||||
estateType2 = "别墅"
|
||||
@@ -228,12 +252,12 @@ class GXDdistrictMatch {
|
||||
}
|
||||
}
|
||||
udata.put("estateType2", estateType2)
|
||||
if(d['AA']){
|
||||
Long lng = d['AA']*10000000000
|
||||
if(d['F']){
|
||||
Long lng = d['F']*10000000000
|
||||
udata.put("lng", lng)
|
||||
}
|
||||
if(d['AB']){
|
||||
Long lat = d['AB'] * 10000000000
|
||||
if(d['E']){
|
||||
Long lat = d['E'] * 10000000000
|
||||
udata.put("lat", lat)
|
||||
}
|
||||
|
||||
@@ -265,7 +289,7 @@ class GXDdistrictMatch {
|
||||
if(!d['U']){
|
||||
d = matchApi(city, region, d, "property", false)
|
||||
if(!d['U']){
|
||||
d = matchApi(city, region, d, null, true)
|
||||
d = matchApi(city, region, d, null, false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,14 +309,18 @@ class GXDdistrictMatch {
|
||||
}
|
||||
|
||||
def matchApi(city, region, d, cat, useMap){
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
|
||||
if(!jsonObject){
|
||||
def jsonObject = null
|
||||
if(!jsonObject && d['H']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, d['H'], cat, useMap)
|
||||
}
|
||||
|
||||
if(!jsonObject && d['C']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, region+d['C'], cat, useMap)
|
||||
if(!jsonObject && d['D']){
|
||||
jsonObject = FangjiaClientUtils.newMatch(city, region+d['D'], cat, useMap)
|
||||
}
|
||||
}
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
if(jsonObject != null) {
|
||||
d['U'] = "接口匹配"
|
||||
d['V'] = jsonObject.get("city")
|
||||
d['W'] = jsonObject.get("district")
|
||||
|
||||
+317
@@ -0,0 +1,317 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class PoiLocation {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||
def commMap = com.fangjia.script.commData.CommonDatautils.queryCommunityByCity()
|
||||
def polylineMap = com.fangjia.script.commData.CommonDatautils.findAllTownRanges()
|
||||
def newAdvMappingMap = com.fangjia.script.commData.CommonDatautils.queryNewTownByCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiLocation();
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def citys = ["莆田市","宁德市","三明市","龙岩市","南平市","漳州市","福州市","厦门市","泉州市","重庆市"]
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def findPois(def city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("run_status", new BasicDBObject('$exists', false))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
if(rets.size()>=10000){
|
||||
index++
|
||||
def datas = checkData(index, rets)
|
||||
// updateToEs(datas)
|
||||
updateToEs2(datas)
|
||||
updateToMG(datas)
|
||||
rets = []
|
||||
print "."
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def checkData(def index, def datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def map = running(d)
|
||||
rets << map
|
||||
if(map.update_status && map.update_status!=100){
|
||||
println "[ ${index} ]"+d.city +" "+totalCount.get() +"/"+rets.size()+" "+map.id+" "+d.district+" 原:"+d.township+" 新:"+map.township
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def regions = getFullRegion(d)
|
||||
def town = null
|
||||
def map =[:]
|
||||
if(d.location){
|
||||
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if(d.district){
|
||||
for(def rs : regions){
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
|
||||
if(regionFlag){
|
||||
//判断township是否在统计局中
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, d.city, d.township)
|
||||
}
|
||||
if(town){
|
||||
if(town.street == d.township){
|
||||
//打标记
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.update_status = 100 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致 100:不修改
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = town.street
|
||||
}
|
||||
}else{
|
||||
def gaodeMap = com.fangjia.script.commData.CommonDatautils.getTownshipInfo(d.location.lon+","+d.location.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]') {
|
||||
def mapTown = getSimilarTown(regions, d.city, gaodeMap.township)
|
||||
if (mapTown) {
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = mapTown.street
|
||||
}else{
|
||||
if(d.district!=gaodeMap.district){
|
||||
map.update_status = 3 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}else{
|
||||
map.update_status = 2 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
//打标记“经纬度有误,不在区域围栏中”
|
||||
map.update_status = 1 //1:区域有问题 2:乡镇街道需修改 3:2边区域不一致
|
||||
map.run_status = 1 // 1:已运行
|
||||
map.id = d._id.toString()
|
||||
map.township = ""
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
def getFullRegion(d){
|
||||
def regions = regionMap.get(d.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.district)){
|
||||
names << r
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
def getSimilarTown(regions, city, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(city+"#"+r.district)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def updateMapPoi(def id, def township){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/didata_map_poi_single/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.township='${township}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"term": {
|
||||
"id": "${id}"
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println(response)
|
||||
}
|
||||
|
||||
|
||||
def updateToEs2(rows) {
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if (map.update_status && map.update_status !=100) {
|
||||
updateMapPoi(map.id, map.township)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateToEs(rows){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int batchSize = 1000;
|
||||
int processedCount = 0;
|
||||
def client = ElasticUtils.getClient89("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
if(map.update_status && map.update_status==2){
|
||||
map.remove("run_status")
|
||||
map.remove("update_status")
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
|
||||
// 批量提交数据
|
||||
if (request.numberOfActions() >= batchSize) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
print "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
|
||||
request = new BulkRequest();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
|
||||
println "结束时间 "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def updateToMG(rows){
|
||||
def rets = []
|
||||
for(def i=0;i<rows.size();i++){
|
||||
def m = rows[i]
|
||||
def udata = new BasicDBObject()
|
||||
m.each{k,v->
|
||||
if(k!="id"){
|
||||
udata.put(k,v)
|
||||
}
|
||||
}
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", m.id)
|
||||
if(udata){
|
||||
MAPPOI_CONN.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println m.toString()+" 无法修改-无值"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class TempExportData {
|
||||
|
||||
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
static main(args) {
|
||||
def task = new TempExportData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run() {
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def rets = []
|
||||
rets << ["ID", "城市", "区域", "名称", "地址", "别名区域", "别名名称", "别名地址", "别名所在小区id"]
|
||||
for (def city : citys) {
|
||||
def cityRets = getResults(city)
|
||||
rets.addAll(cityRets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名称在别名中的小区.xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
def getResults(city) {
|
||||
def rets = []
|
||||
def dists = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "mapping_alias"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
for (def i = 0; i < dists.size(); i++) {
|
||||
def d = dists[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count_.addAndGet(1)
|
||||
try {
|
||||
def objs = CommonDatautils.findSeaweedListByMappingAlias(city, d.region, d.name, "district")
|
||||
if (objs) {
|
||||
def ret = []
|
||||
ret << d._id.toString()
|
||||
ret << d.city
|
||||
ret << d.region
|
||||
ret << d.name
|
||||
ret << d.address
|
||||
|
||||
def str = ""
|
||||
for (def o : objs) {
|
||||
str = str + o.region + "-" + o.name + "-"+o.mapping_alias.toString() + "\n"
|
||||
}
|
||||
ret << str
|
||||
ret << objs.size()
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
println(city +" "+rets.size())
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+105
@@ -0,0 +1,105 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class Test{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
def task = new Test()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def i=1
|
||||
while(i<2){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity("深圳", "district", ["city", "region", "name", "lng", "estate_type2", "name_cat", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
def rets = process(districts)
|
||||
println i + " "+districts.size() +" "+rets.size()
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def process(districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
methodA(rMap)
|
||||
println totalCount.get() +" "+rMap.city+" "+rMap.region+" "+rMap.name
|
||||
rets << rMap.name
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def methodA(rMap) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
// 模拟一个可能会抛出异常的操作
|
||||
riskyOperation();
|
||||
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
System.err.println("[${rMap.name}] Retrying " + retryCount + "/" + MAX_TRY_COUNT);
|
||||
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(10000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
System.err.println("MethodA failed after " + MAX_TRY_COUNT + " attempts.");
|
||||
}
|
||||
}
|
||||
|
||||
// 一个可能会抛出异常的操作
|
||||
def riskyOperation() throws Exception {
|
||||
// 模拟一个有50%概率抛出异常的操作
|
||||
if (Math.random() < 0.5) {
|
||||
throw new Exception("Simulated exception");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+932
@@ -0,0 +1,932 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.client.AggregateIterable
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||
*/
|
||||
class TownMiddleProcess {
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
DBCollection ADMINISTRATIVE_DIVISION_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
DBCollection TOWNPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new TownMiddleProcess();
|
||||
task.run()
|
||||
|
||||
}
|
||||
def run(){
|
||||
//创建省份
|
||||
// createProvince()
|
||||
//创建城市
|
||||
// createCity()
|
||||
//创建区域
|
||||
// createRegion()
|
||||
//创建乡镇街道
|
||||
// createTown()
|
||||
//修改乡镇街道别名
|
||||
// updateTownAlias()
|
||||
//中间表同步到100
|
||||
// synchTown()
|
||||
//添加地级市标志
|
||||
// addCityFlag()
|
||||
// addPinyin()
|
||||
//打标记,将开发区与民政部相同的打上1
|
||||
// addTjjDevFlag()
|
||||
//打标记,用于标识乡镇街道是否输出及格式化输出街道
|
||||
// addStreetFlag()
|
||||
|
||||
// addDistrictDiffFlag()
|
||||
|
||||
//对比中间表的街道与community街道是否一致,因为元极反馈类似“高密市醴泉街道”解析有问题,要去掉“高密市”,然后去了中间表
|
||||
// updateCommunityDiffStreet()
|
||||
// updateTownPoline()
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
|
||||
}
|
||||
|
||||
|
||||
def createProvince(){
|
||||
def cat = "省"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||
|
||||
def tjjKeys = tjjMaps.keySet()
|
||||
def mzbKeys = mzbMaps.keySet()
|
||||
|
||||
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||
def lastRets = []
|
||||
tjjMaps.each {key, val->
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", key)
|
||||
udata.put("tjj_province_code", val)
|
||||
udata.put("mzb_province_code", mzbMaps.get(key))
|
||||
udata.put("tjj_source_year",2023)
|
||||
udata.put("mzb_source_year",2023)
|
||||
lastRets << udata
|
||||
}
|
||||
if(diffKeys){
|
||||
diffKeys.each {k->
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", k)
|
||||
if(tjjMaps.get(k)){
|
||||
udata.put("tjj_province_code", tjjMaps.get(k))
|
||||
udata.put("tjj_source_year",2023)
|
||||
}
|
||||
if(mzbMaps.get(k)){
|
||||
udata.put("mzb_province_code", mzbMaps.get(k))
|
||||
udata.put("mzb_source_year",2023)
|
||||
}
|
||||
lastRets << udata
|
||||
}
|
||||
}
|
||||
saveToDB(lastRets, "省份")
|
||||
}
|
||||
|
||||
|
||||
def createCity(){
|
||||
def cat = "城市"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||
def tjjKeys = tjjMaps.keySet()
|
||||
def mzbKeys = mzbMaps.keySet()
|
||||
def sameKeys = tjjKeys.intersect(mzbKeys)
|
||||
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||
def lastRets = []
|
||||
sameKeys.each {k->
|
||||
def v = tjjMaps.get(k)
|
||||
lastRets << getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||
}
|
||||
|
||||
if(diffKeys){
|
||||
diffKeys.each {k->
|
||||
def v = tjjMaps.get(k)
|
||||
lastRets <<getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||
}
|
||||
}
|
||||
//添加别名
|
||||
lastRets.each {o->
|
||||
def city = RegionData.formatCity(o.city)
|
||||
if(city){
|
||||
def cityObj = CommonDatautils.findAlCityDetail(city)
|
||||
o.put("alias", cityObj.alias)
|
||||
}else{
|
||||
println o.province+" "+o.city +" 房价网无此城市"
|
||||
}
|
||||
}
|
||||
//添加城市围栏
|
||||
def cityPolylineMap = cityPolyline()
|
||||
//添加pid
|
||||
def provPidMap = createPidData("省份")
|
||||
lastRets.each{o->
|
||||
def shapeData = cityPolylineMap.get(o.city)
|
||||
if(shapeData){
|
||||
o.shape = shapeData.shape
|
||||
o.location = shapeData.location
|
||||
}
|
||||
def provId = provPidMap.get(o.province)
|
||||
if(provId){
|
||||
o.pid = provId
|
||||
}
|
||||
}
|
||||
saveToDB(lastRets, cat)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def createRegion(){
|
||||
def cat = "区域"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def mzbMaps = findAllProcincesFromMZB(cat)
|
||||
def tjjKeys = tjjMaps.keySet()
|
||||
def mzbKeys = mzbMaps.keySet()
|
||||
def sameKeys = tjjKeys.intersect(mzbKeys)
|
||||
def diffKeys = (tjjKeys - mzbKeys) + (mzbKeys - tjjKeys)
|
||||
def lastRets = []
|
||||
sameKeys.each {k->
|
||||
def v = tjjMaps.get(k)
|
||||
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||
// ud.source = "tjj|mzb"
|
||||
lastRets << ud
|
||||
}
|
||||
|
||||
if(diffKeys){
|
||||
diffKeys.each {k->
|
||||
def v = tjjMaps.get(k)
|
||||
def ud = getAdministrativeCodeInfo(k, v, mzbMaps)
|
||||
// if(v){
|
||||
// ud.source = "tjj"
|
||||
// }else{
|
||||
// ud.source = "mzb"
|
||||
// }
|
||||
lastRets << ud
|
||||
}
|
||||
}
|
||||
//添加别名
|
||||
def regionMap = CommonDatautils.getAllCityFullRegions()
|
||||
lastRets.each {o->
|
||||
if(o.district.contains("开发区")){
|
||||
println "ssss"
|
||||
}
|
||||
def city = RegionData.formatCity(o.city)
|
||||
def district = RegionData.formatRegion(city, o.district)
|
||||
if(city && district){
|
||||
def cMap = regionMap.get(city)
|
||||
if(cMap){
|
||||
def regionInfoMap = cMap.get(district)
|
||||
if(regionInfoMap){
|
||||
def regions = regionInfoMap.get("pattern")
|
||||
o.put("alias", regions)
|
||||
o.put("mzb_district_name", regionInfoMap.full_region)
|
||||
if(!o.mzb_district_code && regionInfoMap.full_region){
|
||||
def mzb_val_str = mzbMaps.get(o.province+"|@|"+o.city+"|@|"+regionInfoMap.full_region)
|
||||
if(mzb_val_str){
|
||||
def mzb_vals = mzb_val_str.split("\\|@\\|")
|
||||
o.put("mzb_province_code", mzb_vals[0])
|
||||
o.put("mzb_city_code", mzb_vals[1])
|
||||
o.put("mzb_district_code", mzb_vals[2])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}else{
|
||||
println o.province+" "+o.city+" "+ o.district +" 房价网无此城市"
|
||||
}
|
||||
}
|
||||
|
||||
//添加pid
|
||||
def provPidMap = createPidData("城市")
|
||||
|
||||
//添加区域围栏
|
||||
def regionPolylineMap = regionPolyline()
|
||||
|
||||
lastRets.each{o->
|
||||
def shapeData = regionPolylineMap.get(o.city+"|@|"+o.district)
|
||||
if(shapeData){
|
||||
o.shape = shapeData.shape
|
||||
o.location = shapeData.location
|
||||
}
|
||||
def provId = provPidMap.get(o.city)
|
||||
if(provId){
|
||||
o.pid = provId
|
||||
}
|
||||
}
|
||||
//是否为县级市
|
||||
saveToDB(lastRets, cat)
|
||||
}
|
||||
|
||||
def createTown(){
|
||||
def cat = "乡镇街道"
|
||||
def tjjMaps = findAllProcincesFromTJJ(cat)
|
||||
def lastRets = []
|
||||
def govMap = findAllGovTown()
|
||||
tjjMaps.each {k,v->
|
||||
def ud = getAdministrativeCodeInfo(k, v, null)
|
||||
if(govMap && govMap.containsKey(k)){
|
||||
ud.put("has_gov", govMap.get(k))
|
||||
}
|
||||
lastRets << ud
|
||||
}
|
||||
|
||||
//添加区域围栏
|
||||
def regionPolylineMap = townPolyline()
|
||||
//添加pid
|
||||
def provPidMap = createPidData("区域")
|
||||
|
||||
lastRets.each{o->
|
||||
def shapeData = getStreetPolyline(regionPolylineMap, o, o.district)
|
||||
if(!shapeData){
|
||||
def newRegion = o.district
|
||||
def standardRegion = RegionData.formatRegion(o.city,o.district)
|
||||
if(standardRegion && standardRegion!=newRegion){
|
||||
newRegion = standardRegion
|
||||
shapeData = getStreetPolyline(regionPolylineMap, o, newRegion)
|
||||
}
|
||||
}
|
||||
if(shapeData){
|
||||
o.shape = shapeData.shape
|
||||
if(shapeData.location){
|
||||
o.location = shapeData.location
|
||||
}
|
||||
}
|
||||
def provId = provPidMap.get(o.city+"@"+o.district)
|
||||
if(provId){
|
||||
o.pid = provId
|
||||
}
|
||||
}
|
||||
//是否为县级市
|
||||
saveToDB(lastRets, cat)
|
||||
}
|
||||
|
||||
def getStreetPolyline(regionPolylineMap, o, district){
|
||||
def shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+o.street)
|
||||
if(!shapeData){
|
||||
def streetKey = o.street
|
||||
if(streetKey.toString().endsWith("镇")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"街道"
|
||||
}else if(streetKey.toString().endsWith("乡")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇"
|
||||
}else if(streetKey.toString().endsWith("街道")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"镇"
|
||||
}else if(streetKey.toString().endsWith("地区")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("地区"))+"镇"
|
||||
}
|
||||
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
|
||||
if(!shapeData){
|
||||
streetKey = o.street
|
||||
if(streetKey.toString().endsWith("镇")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"乡"
|
||||
}else if(streetKey.toString().endsWith("乡")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))+"镇"
|
||||
}else if(streetKey.toString().endsWith("街道")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"社区"
|
||||
}
|
||||
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
|
||||
}
|
||||
if(!shapeData){
|
||||
streetKey = o.street
|
||||
if(streetKey.toString().endsWith("镇")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))+"地区"
|
||||
}else if(streetKey.toString().endsWith("街道")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))+"乡"
|
||||
}
|
||||
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
|
||||
}
|
||||
if(!shapeData){
|
||||
if(streetKey.toString().endsWith("镇")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("镇"))
|
||||
}else if(streetKey.toString().endsWith("乡")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("乡"))
|
||||
}else if(streetKey.toString().endsWith("街道")){
|
||||
streetKey = streetKey.toString().substring(0, streetKey.toString().lastIndexOf("街道"))
|
||||
}
|
||||
shapeData = regionPolylineMap.get(o.city+"|@|"+ district+"|@|"+streetKey)
|
||||
}
|
||||
}
|
||||
return shapeData
|
||||
}
|
||||
|
||||
|
||||
def getAdministrativeCodeInfo(key, val, mzbMaps){
|
||||
def udata = new BasicDBObject()
|
||||
def keys = key.split("\\|@\\|")
|
||||
def vals = null
|
||||
if(val){
|
||||
vals = val.split("\\|@\\|")
|
||||
}
|
||||
|
||||
def mzb_vals = null
|
||||
if(mzbMaps && mzbMaps.containsKey(key)){
|
||||
def mzb_val_str = mzbMaps.get(key)
|
||||
mzb_vals = mzb_val_str.split("\\|@\\|")
|
||||
}
|
||||
if(keys.size()>=1){
|
||||
udata.put("province", keys[0])
|
||||
if(vals && vals.size()>=1){
|
||||
udata.put("tjj_province_code", vals[0])
|
||||
udata.put("tjj_source_year",2023)
|
||||
}
|
||||
if(mzb_vals && mzb_vals.size()>=1){
|
||||
udata.put("mzb_province_code", mzb_vals[0])
|
||||
udata.put("mzb_source_year",2023)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(keys.size()>=2){
|
||||
udata.put("city", keys[1])
|
||||
if(vals && vals.size()>=2){
|
||||
udata.put("tjj_city_code", vals[1])
|
||||
}
|
||||
if(mzb_vals && mzb_vals.size()>=2){
|
||||
udata.put("mzb_city_code", mzb_vals[1])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(keys.size()>=3){
|
||||
udata.put("district", keys[2])
|
||||
if(vals && vals.size()>=3) {
|
||||
udata.put("tjj_district_code", vals[2])
|
||||
}
|
||||
if(mzb_vals && mzb_vals.size()>=3){
|
||||
udata.put("mzb_district_code", mzb_vals[2])
|
||||
}
|
||||
}
|
||||
|
||||
if(keys.size()>=4){
|
||||
udata.put("street", keys[3])
|
||||
if(vals && vals.size()>=4) {
|
||||
udata.put("tjj_street_code", vals[3])
|
||||
}
|
||||
if(mzb_vals && mzb_vals.size()>=4){
|
||||
udata.put("mzb_street_code", mzb_vals[3])
|
||||
}
|
||||
}
|
||||
|
||||
return udata
|
||||
}
|
||||
|
||||
|
||||
def updateTownAlias(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇别名问题审核交付_0418.xlsx",{ row->
|
||||
if(row['G'] && row['G']=="不处理"){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city",row['C'])
|
||||
q.put("district",row['D'])
|
||||
q.put("street",row['E'])
|
||||
def d = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(d){
|
||||
def alias = []
|
||||
alias << row['F']
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}else{
|
||||
println row['C']+" "+row['D']+" "+row["E"]
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
def synchTown(){
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000)
|
||||
def datas = []
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
if (datas.size() >= 2000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
println datas.size()
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def saveToDB(lastRets, type){
|
||||
def dataList = []
|
||||
for(def ret : lastRets){
|
||||
ret.put("c_date", new Date())
|
||||
ret.put("m_date", new Date())
|
||||
ret.put("type", type)
|
||||
dataList << ret
|
||||
|
||||
if (dataList.size() >= 5000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
|
||||
dataList.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (dataList.size() > 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL.insert(dataList);
|
||||
dataList.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def createPidData(def type){
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject("type", type)).batchSize(5000)
|
||||
def map = [:]
|
||||
def pids = []
|
||||
def step = 2
|
||||
if(type == "城市"){
|
||||
step = 4
|
||||
}else if(type == "区域"){
|
||||
step = 6
|
||||
}
|
||||
for(def d : dbcursor){
|
||||
def pid = null
|
||||
def code = null
|
||||
if(d.mzb_province_code){
|
||||
if(type == "省份") {
|
||||
code = d.mzb_province_code.toString()
|
||||
}else if(type == "城市"){
|
||||
code = d.mzb_city_code.toString()
|
||||
}else if(type == "区域"){
|
||||
code = d.mzb_district_code.toString()
|
||||
}
|
||||
}else if(d.tjj_province_code){
|
||||
if(type == "省份") {
|
||||
code = d.tjj_province_code.toString()
|
||||
}else if(type == "城市"){
|
||||
code = d.tjj_city_code.toString()
|
||||
}else if(type == "区域"){
|
||||
code = d.tjj_district_code.toString()
|
||||
}
|
||||
}
|
||||
|
||||
pid = code.substring(0,step)
|
||||
if(pids.contains(d.city+"-"+pid)){
|
||||
pid = code
|
||||
}
|
||||
if(!pids.contains(d.city+"-"+pid)){
|
||||
pids << d.city+"-"+pid
|
||||
}
|
||||
if(type == "省份"){
|
||||
map.put(d.province, pid)
|
||||
}else if(type == "城市"){
|
||||
map.put(d.city, pid)
|
||||
}else if(type == "区域"){
|
||||
map.put(d.city+"@"+d.district, pid)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def findAllGovTown(){
|
||||
def q = new BasicDBObject();
|
||||
q.put("has_gov", new BasicDBObject('$ne', ''))
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province_name",1)
|
||||
keys.put("city_name",1)
|
||||
keys.put("area_name", 1)
|
||||
keys.put("street_name", 1)
|
||||
keys.put("has_gov", 1)
|
||||
def dbcursor = COMMUNITY_CONN.find(q,keys).batchSize(5000)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.province_name+"|@|"+d.city_name+"|@|"+d.area_name+"|@|"+d.street_name
|
||||
if(!map.containsKey(key)){
|
||||
map.put(key, d.has_gov)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findAllProcincesFromTJJ(cat){
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
def match = null
|
||||
if(cat=="省"){
|
||||
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code'))
|
||||
}else if(cat=="城市"){
|
||||
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code'))
|
||||
}else if(cat=="区域"){
|
||||
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code'))
|
||||
}else if(cat=="乡镇街道"){
|
||||
groupFields.put("_id", new BasicDBObject("province_name", '$province_name').append("province_code", '$province_code').append("city_name", '$city_name').append("city_code", '$city_code').append("area_name", '$area_name').append("area_code", '$area_code').append("street_name", '$street_name').append("street_code", '$street_code'))
|
||||
}
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
|
||||
// 执行聚合查询
|
||||
AggregationOutput output = COMMUNITY_CONN.aggregate(Arrays.asList(group));
|
||||
|
||||
def map = [:]
|
||||
// 遍历结果
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
if(obj.get("city_name").toString()!="省直辖县级行政区划"){
|
||||
def key = obj.get("province_name")
|
||||
def value = obj.get("province_code")
|
||||
if(cat=="城市"){
|
||||
key = key +"|@|"+ obj.get("city_name")
|
||||
value = value +"|@|"+ obj.get("city_code")
|
||||
}else if(cat=="区域"){
|
||||
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")
|
||||
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")
|
||||
}else if(cat=="乡镇街道"){
|
||||
key = key +"|@|"+ obj.get("city_name")+"|@|"+ obj.get("area_name")+"|@|"+ obj.get("street_name")
|
||||
value = value +"|@|"+ obj.get("city_code")+"|@|"+ obj.get("area_code")+"|@|"+ obj.get("street_code")
|
||||
}
|
||||
int count = (Integer) result.get("count");
|
||||
map.put(key, value)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
def findAllProcincesFromMZB(cat){
|
||||
|
||||
BasicDBObject match = null;
|
||||
// 定义聚合管道,按 city 字段分组,并计算每个城市的数量
|
||||
DBObject groupFields = new BasicDBObject();
|
||||
|
||||
if(cat == "省"){
|
||||
match = new BasicDBObject('$match', new BasicDBObject("level", "省份"));
|
||||
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code'))
|
||||
}else if(cat == "城市"){
|
||||
match = new BasicDBObject('$match', new BasicDBObject("level", "城市"));
|
||||
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code'))
|
||||
}else if(cat == "区域"){
|
||||
match = new BasicDBObject('$match', new BasicDBObject("level", "区域"));
|
||||
groupFields.put("_id", new BasicDBObject("province", '$province').append("mca_province_code", '$mca_province_code').append("city",'$city').append("mca_city_code",'$mca_city_code').append("district",'$district').append("mca_district_code",'$mca_district_code'))
|
||||
}
|
||||
|
||||
|
||||
groupFields.put("count", new BasicDBObject('$sum', 1));
|
||||
DBObject group = new BasicDBObject('$group', groupFields);
|
||||
|
||||
// 执行聚合查询
|
||||
AggregationOutput output = ADMINISTRATIVE_DIVISION_CONN.aggregate(Arrays.asList(match, group));
|
||||
def map = [:]
|
||||
// 遍历结果
|
||||
for (DBObject result : output.results()) {
|
||||
def obj = (DBObject ) result.get("_id");
|
||||
if(obj.get("city").toString()!="省直辖县级行政区划"){
|
||||
def key = obj.get("province")
|
||||
def value = obj.get("mca_province_code")
|
||||
if(cat == "城市"){
|
||||
key = key +"|@|"+obj.get("city")
|
||||
value = value + "|@|" + obj.get("mca_city_code")
|
||||
}else if(cat == "区域"){
|
||||
key = key +"|@|"+obj.get("city")+"|@|"+obj.get("district")
|
||||
value = value + "|@|" + obj.get("mca_city_code")+ "|@|" + obj.get("mca_district_code")
|
||||
}
|
||||
int count = (Integer) result.get("count");
|
||||
map.put(key, value)
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def cityPolyline(){
|
||||
def dbcursor = CITYPOLYLINE_COLL.find(new BasicDBObject())
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.name+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
map.put(d.name, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def regionPolyline(){
|
||||
def dbcursor = REGIONPOLYLINE_COLL.find(new BasicDBObject("level","district"))
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
map.put(fullCity.city+"|@|"+d.region, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def townPolyline(){
|
||||
def dbcursor = TOWNPOLYLINE_COLL.find(new BasicDBObject()).batchSize(5000)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
def center = d.center
|
||||
if(center){
|
||||
def pointArrs = center.split(",")
|
||||
udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()])
|
||||
}
|
||||
map.put(d.city+"|@|"+d.fjw_region+"|@|"+d.town, udata)
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
def addCityFlag(){
|
||||
// ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row->
|
||||
// if(row['A']){
|
||||
// def q = new BasicDBObject("type", "城市")
|
||||
// q.put("city", row['A'])
|
||||
// def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
// if(obj){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("prefecture_level_city", "是")
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
// }else{
|
||||
// println row['A']+" 没找到该城市"
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\地级市.xlsx",{ row->
|
||||
if(row['A']){
|
||||
def q = new BasicDBObject("type", "区域")
|
||||
q.put("district", row['A'])
|
||||
def count = ADMINISTRATIVE_MAPPING_COLL.count(q)
|
||||
if(count && count == 1){
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("county_level_city", "是")
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata))
|
||||
}else{
|
||||
println row['A']+" 没找到该城市"
|
||||
}
|
||||
}
|
||||
},1);
|
||||
}
|
||||
|
||||
def addPinyin(){
|
||||
def q = new BasicDBObject("type", "乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||
for(def d : dbcursor){
|
||||
if(d.street){
|
||||
def street_pinyin = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.street, " ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_pinyin", street_pinyin)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def addTjjDevFlag(){
|
||||
def q = new BasicDBObject("type", "区域")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||
for(def d : dbcursor){
|
||||
if(d.district && d.mzb_district_name && d.district==d.mzb_district_name){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("district_diff", 1) //民政部,统计局 名称相同 district_diff打1
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addStreetFlag(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道-结果.xlsx",{ row->
|
||||
if(row['I']){
|
||||
if(row['I']=="标0"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_output", 0) //0:不输出 1:输出
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
}else if(row['I']=="标1"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_output", 1) //0:不输出 1:输出
|
||||
if(row['G']){
|
||||
udata.put("street", row['G']) //街道中去除区域
|
||||
}
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
def addDistrictDiffFlag(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\区域-结果.xlsx",{ row->
|
||||
def flag = row['F'].toString().toDouble().toInteger()
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("district_diff", flag)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",new ObjectId(row['A'])), new BasicDBObject('$set',udata))
|
||||
});
|
||||
}
|
||||
|
||||
def updateCommunityDiffStreet(){
|
||||
def q = new BasicDBObject("type", "乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q).batchSize(2000)
|
||||
for(def d : dbcursor){
|
||||
def query = new BasicDBObject()
|
||||
query.put("city_name", d.city)
|
||||
query.put("area_name", d.district)
|
||||
query.put("street_name", d.street)
|
||||
def obj = COMMUNITY_CONN.findOne(query)
|
||||
if(!obj){
|
||||
query.put("street_name", d.district+d.street)
|
||||
obj = COMMUNITY_CONN.findOne(query)
|
||||
if(obj){
|
||||
def newStreet = obj.street_name.replace(d.district, "")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street_name", newStreet)
|
||||
COMMUNITY_CONN.update(query, new BasicDBObject('$set', udata), false, true)
|
||||
println d.city +" "+d.district+" "+d.street +" community: [${obj.street_name}] new_street:${newStreet}"
|
||||
}else{
|
||||
println d.city +" "+d.district+" "+d.street
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def updateTownPoline(){
|
||||
def city = "唐山市"
|
||||
def name = "高新技术开发区街道"
|
||||
def q = new BasicDBObject("city", city).append("town","唐山高新技术产业开发区")
|
||||
println city+" "+ name+" "+TOWNPOLYLINE_COLL.count(q)
|
||||
def d = TOWNPOLYLINE_COLL.findOne(q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
println d.city+" "+d.region+" "+d.town+" "+rets.size()
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
|
||||
|
||||
def updateQuery = new BasicDBObject("city", city).append("street",name)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
def checkTown(){
|
||||
def q = new BasicDBObject("shape", new BasicDBObject('$exists', false)).append("type","乡镇街道")
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def ad : dbcursor){
|
||||
def polyline_q = new BasicDBObject("city", ad.city).append("town",ad.district+ad.street)
|
||||
def count = TOWNPOLYLINE_COLL.count(polyline_q)
|
||||
if(count==1){
|
||||
println ad.city+" "+ ad.street+" "+count
|
||||
|
||||
def d = TOWNPOLYLINE_COLL.findOne(polyline_q)
|
||||
if(d){
|
||||
def resultMap = [:]
|
||||
resultMap.type = "polygon"
|
||||
def arrs = d.polyline.split("\\|")
|
||||
def rets = []
|
||||
for(def s : arrs){
|
||||
def points = s.split(";")
|
||||
def pointList = []
|
||||
for(def pointStr : points){
|
||||
def pointArr = pointStr.split(",")
|
||||
pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()]
|
||||
}
|
||||
rets << pointList
|
||||
}
|
||||
resultMap.coordinates = rets
|
||||
if(rets.size()>1) {
|
||||
resultMap.type = "MultiPolygon"
|
||||
}
|
||||
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("shape", resultMap)
|
||||
udata.put("polyline_source", "区域+街道匹配")
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", ad._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def updateTownshipByRegion(){
|
||||
def q = new BasicDBObject("mzb_district_name", new BasicDBObject('$exists', true)).append("type","区域")
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("mzb_district_name",1)
|
||||
keys.put("mzb_district_code",1)
|
||||
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys)
|
||||
for(def d : dbcursor){
|
||||
def upquery = new BasicDBObject("type","乡镇街道").append("district", d.district).append("city",d.city)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("mzb_district_name",d.mzb_district_name)
|
||||
udata.put("mzb_district_code",d.mzb_district_code)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
+324
@@ -0,0 +1,324 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class TownshipStrandUpdate {
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
def fullCityMap = null
|
||||
def regionMap = null
|
||||
def commMap = null
|
||||
def polylineMap = null
|
||||
def newAdvMappingMap = null
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
task.process()
|
||||
|
||||
}
|
||||
|
||||
|
||||
void process() {
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
polylineMap = CommonDatautils.findAllTownRanges()
|
||||
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
|
||||
// def citys = ["唐山","邯郸","石家庄","保定","秦皇岛","廊坊","沧州","张家口","衡水","邢台","承德"]//CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
if(["中山"].contains(city)){
|
||||
continue
|
||||
}
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\小区\\小区乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "district", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
dists = checkData(dists)
|
||||
def datas = createDatas(dists)
|
||||
ExcelUtils.write(fileName, datas, "${city}")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def checkData(def dists){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def d = dists[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
rets << running(d)
|
||||
println d.city +" "+totalCount.get() +"/"+dists.size()+" "+d.name+" "
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return dists
|
||||
}
|
||||
|
||||
def running(d){
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
def gaodeMap = null
|
||||
def town = null
|
||||
def mapTown = null
|
||||
def oldSimlarTown = null
|
||||
def newSimlarTown = null
|
||||
if(d.lng){
|
||||
d.lng = d.lng/10000000000.0
|
||||
d.lat = d.lat/10000000000.0
|
||||
if(d.township){
|
||||
town = getSimilarTown(regions, fullCity, d.township)
|
||||
}
|
||||
|
||||
if(town && town.shape && town.shape.coordinates){
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(town.shape.coordinates))
|
||||
if(flag){
|
||||
d.polygon_town = town.street
|
||||
}
|
||||
}
|
||||
d.strand_street = ""
|
||||
gaodeMap = getTownshipInfo(d.lng+","+d.lat)
|
||||
if(gaodeMap && gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTown = getSimilarTown(regions, fullCity, gaodeMap.township)
|
||||
if(mapTown){
|
||||
d.strand_street = mapTown.street
|
||||
if(mapTown.shape && mapTown.shape.coordinates){
|
||||
d.map_town_shape = "有"
|
||||
def flag = InsidePolygon.isInArea(d.lng, d.lat, InsidePolygon.createPoint4(mapTown.shape.coordinates))
|
||||
if(flag){
|
||||
d.map_polygon_town = mapTown.street
|
||||
}
|
||||
}
|
||||
}else{
|
||||
oldSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, polylineMap.get(fullCity.city))
|
||||
if(oldSimlarTown){
|
||||
d.old_similar_town = oldSimlarTown.region+" "+oldSimlarTown.town
|
||||
d.old_similar_town_score = oldSimlarTown.score
|
||||
}
|
||||
newSimlarTown = getSimilarTownByStr(fullCity, gaodeMap.township, newAdvMappingMap.get(fullCity.city))
|
||||
if(newSimlarTown){
|
||||
d.new_similar_town = newSimlarTown.region+" "+newSimlarTown.town
|
||||
d.new_similar_town_score = newSimlarTown.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
def mapCity = null
|
||||
def mapRegion = null
|
||||
def mapTownship = null
|
||||
def mapAddr = null
|
||||
if(gaodeMap){
|
||||
if(gaodeMap.city && gaodeMap.city!="[]"){
|
||||
mapCity = gaodeMap.city
|
||||
}
|
||||
if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
mapRegion = gaodeMap.district
|
||||
}
|
||||
if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTownship = gaodeMap.township
|
||||
}
|
||||
if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
|
||||
mapAddr = gaodeMap.format_addr
|
||||
}
|
||||
}
|
||||
d.mapCity = mapCity
|
||||
d.mapRegion = mapRegion
|
||||
d.mapTownship = mapTownship
|
||||
d.mapAddr = mapAddr
|
||||
if(mapCity && mapRegion){
|
||||
if(mapCity == fullCity.city){
|
||||
if(regions.contains(mapRegion)){
|
||||
if(mapTownship){
|
||||
if(town){
|
||||
d.old_tjj_township = "是"
|
||||
}else{
|
||||
d.old_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTown){
|
||||
d.new_tjj_township = "是"
|
||||
}else{
|
||||
d.new_tjj_township = "否"
|
||||
}
|
||||
|
||||
if(mapTownship == d.township){
|
||||
if(town){
|
||||
d.remark = "相同且为统计局乡镇"
|
||||
}else{
|
||||
d.remark = "非统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}else{
|
||||
if(mapTown){
|
||||
d.remark = "更正"
|
||||
}else{
|
||||
d.remark = "不同且不为统计局乡镇"
|
||||
d.opt = "原因很多:可能存在此乡镇,但区域不同,也有可能不存在此乡镇,也有可能与其它镇合并"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
d.remark = "地图无乡镇街道"
|
||||
d.opt = "地图无街道信息"
|
||||
}
|
||||
|
||||
}else{
|
||||
d.remark = "区域不同"
|
||||
d.opt = "经纬度可能跨区域"
|
||||
}
|
||||
}else{
|
||||
d.remark = "城市不同"
|
||||
d.opt = "经纬度可能跨城市"
|
||||
}
|
||||
}
|
||||
|
||||
if(d.map_town_shape && d.map_town_shape=="有"){
|
||||
if(!d.map_polygon_town){
|
||||
d.opt = "经纬度可疑"
|
||||
}
|
||||
}
|
||||
return d;
|
||||
}
|
||||
|
||||
def getSimilarTownByStr(fullCity, townstr, townList){
|
||||
|
||||
def maxScore = 0
|
||||
def town = null
|
||||
for(def townObj : townList){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(townObj.town,townstr);
|
||||
if(score > maxScore){
|
||||
maxScore = score
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
town.score = maxScore
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getSimilarTown(regions, fullCity, townstr){
|
||||
def town = null
|
||||
for(def r : regions){
|
||||
def townList = commMap.get(fullCity.city+"#"+r)
|
||||
|
||||
for(def townObj : townList){
|
||||
if(townObj.street.contains(townstr) || townstr.contains(townObj.street)){
|
||||
town = townObj
|
||||
}else if(townObj.alias && townObj.alias.contains(townstr)){
|
||||
town = townObj
|
||||
}else if(townObj.street.toString().endsWith("乡")||townObj.street.toString().endsWith("镇")||townObj.street.toString().endsWith("街道")||townObj.street.toString().endsWith("社区")||townObj.street.toString().endsWith("地区")){
|
||||
def newTown = townObj.street.replace("乡","").replace("镇","").replace("街道","").replace("社区","").replace("地区","")
|
||||
if(townstr.contains(newTown)){
|
||||
town = townObj
|
||||
}
|
||||
}
|
||||
if(town){
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return town
|
||||
}
|
||||
|
||||
|
||||
def getFullRegion(fullCity, d){
|
||||
def regions = regionMap.get(fullCity.city)
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.region)){
|
||||
names << r.district
|
||||
}
|
||||
}
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
|
||||
def createDatas(dists){
|
||||
def rets = []
|
||||
rets << ["城市","区域","名称","地址","物业类型","乡镇","经度","纬度","原始乡镇是否在围栏中","经纬度是否在围栏中","地图城市","地图区域","地图乡镇","地图乡镇是否在围栏中","地图乡镇是否在统计局中","地图所在乡镇街道是否有围栏","地图地址","标准镇","备注", "操作","原始围栏表相似镇", "原始围栏表相似得分","中间表相似镇", "中间表相似得分"]
|
||||
for(def d : dists){
|
||||
rets << [d.city,d.region,d.name,d.address,d.estate_type2,d.township,d.lng,d.lat, d.old_tjj_township, d.polygon_town, d.mapCity,d.mapRegion,d.mapTownship,d.map_polygon_town,d.new_tjj_township,d.map_town_shape,d.mapAddr,d.strand_street, d.remark, d.opt, d.old_similar_town, d.old_similar_town_score, d.new_similar_town, d.new_similar_town_score]
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
if(jsonObject && jsonObject.regeocode) {
|
||||
def addressComponent = jsonObject.regeocode.addressComponent
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
if(addressComponent.city.size()>0){
|
||||
city = addressComponent.city
|
||||
}else{
|
||||
if(addressComponent.province && (addressComponent.province.endsWith("省") || addressComponent.province.endsWith("自治区"))){
|
||||
city = addressComponent.district
|
||||
}else{
|
||||
city = addressComponent.province
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
Thread.sleep(3000)
|
||||
print "."
|
||||
}
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -1,13 +1,46 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class TradeDatasDistBatch {
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
|
||||
|
||||
public static final DBCollection alias_fang_COLL = MongoUtils.getCollectionByName("business", "alias_fang", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
// def
|
||||
def task = new TradeDatasDistBatch()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\房天下成交别名.xlsx",{d->
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("city", d['A'])
|
||||
udata.put("region", d['B'])
|
||||
udata.put("name", d['C'])
|
||||
udata.put("alias", d['D'])
|
||||
alias_fang_COLL.save(udata)
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def flag = null
|
||||
// if(!dist){
|
||||
// def rets = CommonDatautils.findDistrictByAlias(d['A'], d['B'], d['C'], "district")
|
||||
// if(rets && rets.size()==1){
|
||||
// flag = "别名匹配"
|
||||
// dist = rets[0]
|
||||
// }
|
||||
// }else{
|
||||
// flag = "名称匹配"
|
||||
// }
|
||||
// if(flag){
|
||||
// d['F'] = dist.city
|
||||
// d['G'] = dist.region
|
||||
// d['H'] = dist.name
|
||||
// }
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+24
-4
@@ -135,6 +135,26 @@ class FangjiaClientUtils {
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def format(def city, def address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
try {
|
||||
if(address){
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
}
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
}
|
||||
return jsonObj;
|
||||
}
|
||||
|
||||
public static def arroundPrice(city,address){
|
||||
def jsonObj = null
|
||||
def content = null
|
||||
@@ -228,9 +248,9 @@ class FangjiaClientUtils {
|
||||
def url = null;
|
||||
try {
|
||||
address = address.replace("{","").replace("\\","")
|
||||
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
if(city){
|
||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
@@ -334,13 +354,13 @@ class FangjiaClientUtils {
|
||||
def content = null
|
||||
try {
|
||||
address = address.replace("{","")
|
||||
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
|
||||
def url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true"
|
||||
if(city){
|
||||
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
content = new HttpRequest().newGet(url, null, "utf-8")
|
||||
jsonObj = JSONObject.fromObject(content)
|
||||
println jsonObj
|
||||
// println jsonObj
|
||||
} catch (Exception e) {
|
||||
println address +" "+content
|
||||
e.printStackTrace();
|
||||
|
||||
+138
@@ -0,0 +1,138 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 房价网内部模型估值项目
|
||||
* 小区建筑类型数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataCommbuildtypeBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataCommbuildtypeBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rets = []
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
for(def city : citys){
|
||||
def distIds = FjwPgsqlCommUtils.findAllDistrictIdsByOracle(cityCodeDatas.get(city))
|
||||
|
||||
def rows = createDatas(cityCodeDatas.get(city), distIds)
|
||||
rets.addAll(rows)
|
||||
println city +" "+rows.size()
|
||||
}
|
||||
|
||||
delPgData()
|
||||
saveToPgSql(rets)
|
||||
}
|
||||
|
||||
def createDatas(cityCode, dists){
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def id = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def dist = CommonDatautils.findDistrictFieldById(id, ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
|
||||
if(dist){
|
||||
def ret = createData(cityCode, dist)
|
||||
if(ret){
|
||||
rets.addAll(ret)
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(cityCode, dist){
|
||||
|
||||
def types = CcbCodeUtils.parseBuildType(dist.build_type)
|
||||
def rets = []
|
||||
for(def type : types){
|
||||
def ret = []
|
||||
ret << cityCode
|
||||
ret << dist._id.toString()
|
||||
ret << type
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
|
||||
rets << ret
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def delPgData(){
|
||||
sqlInstance.execute("delete from base.bigdata_commbuildtype")
|
||||
println "base.bigdata_commbuildtype 数据已删除"
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
def sqls = []
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
def sql = "insert into base.bigdata_commbuildtype values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
println "start: "+new Date()
|
||||
def index =0
|
||||
sqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
index++
|
||||
println index
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14])
|
||||
}
|
||||
}
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
+370
@@ -0,0 +1,370 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 房价网内部估值项目使用
|
||||
* 小区数据同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
//"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
|
||||
//"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
|
||||
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
}
|
||||
|
||||
if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
if(!fangjiaDists.contains(ccbId)){
|
||||
def datas = ccbMaps.get(ccbId)
|
||||
datas[datas.size()-1] = 1
|
||||
rets << datas
|
||||
}
|
||||
}
|
||||
println "添加删除状态小区:"+rets.size()
|
||||
return rets
|
||||
}
|
||||
|
||||
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
|
||||
//def rets = Collections.synchronizedList(new ArrayList());
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def ret = createData(cityCodeDataMap, distCodeMap, dist)
|
||||
if(ret){
|
||||
rets << ret
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData(def cityCodeDataMap, def distCodeMap, def dist){
|
||||
def ret = []
|
||||
try{
|
||||
def data = distCodeMap.get(dist.region)
|
||||
if(!data){
|
||||
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret << cityCodeDataMap.get(dist.city)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist._id.toString()
|
||||
if(data){
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.name
|
||||
ret << dist.address
|
||||
if(dist.lat){
|
||||
def lat = dist.lat/10000000000d
|
||||
def lng = dist.lng/10000000000d
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << dist.all_area
|
||||
ret << dist.build_area
|
||||
ret << dist.virescence_percent
|
||||
ret << dist.cubage_percent
|
||||
ret << dist.car_site_count
|
||||
ret << CcbCodeUtils.parseBuildYear(dist.finish_building_date)
|
||||
ret << CcbCodeUtils.parseHouseUse(dist.estate_type2)
|
||||
ret << dist.household_count
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd").toInteger()
|
||||
ret << dist.status
|
||||
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "pa101"
|
||||
ret << ""
|
||||
if(dist.address){
|
||||
if(dist.street){
|
||||
ret << dist.street
|
||||
if(dist.street_number){
|
||||
ret << dist.street_number
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
// def jsonObj = FangjiaClientUtils.addressStructure(dist.city, dist.address)
|
||||
// if(jsonObj && jsonObj.result && jsonObj.result.street){
|
||||
// if(jsonObj.result.street){
|
||||
// ret << jsonObj.result.street
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
//
|
||||
// if(jsonObj.result.streetNumber){
|
||||
// ret << jsonObj.result.streetNumber
|
||||
// }else{
|
||||
// ret << ""
|
||||
// }
|
||||
// }else{
|
||||
// ret << ""
|
||||
// ret << ""
|
||||
// }
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << 0
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << 0
|
||||
}catch(Exception e){
|
||||
println dist.city + " "+dist.name+" error"
|
||||
e.printStackTrace()
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
public def findPgCcbDistrict(cityid){
|
||||
def sql = "select * from base.bigdata_community where urbn_id='${cityid}'";
|
||||
def map = [:]
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def data = []
|
||||
data << r.urbn_id
|
||||
data << r.gujia_admndiv_id
|
||||
data << r.gujia_comm_id
|
||||
data << r.gujia_admndiv_nm
|
||||
data << r.comm_nm
|
||||
data << r.gujia_dtl_adr
|
||||
data << r.gujia_bds_ltt
|
||||
data << r.gujia_bds_lgt
|
||||
data << r.gujiabdsocpoldtotarea
|
||||
data << r.gujiabds_bld_tot_area
|
||||
data << r.gujia_bds_grnrate
|
||||
data << r.gujia_bds_vol_rate
|
||||
data << r.gujia_bds_park_num
|
||||
data << r.gujia_bds_bld_yr_rc
|
||||
data << r.gujia_bds_hs_use_val
|
||||
data << r.totunits
|
||||
data << r.gujia_bds_data_tm_val
|
||||
data << r.gujiabds_udt_flag_val
|
||||
data << r.data_udt_tm
|
||||
data << r.gujia_comm_tp_val
|
||||
data << r.gujia_comm_fmnm
|
||||
data << r.gujia_str
|
||||
data << r.gujia_hsno
|
||||
data << r.gujiawthr_new_pan_ind
|
||||
data << r.gujia_comm_intd
|
||||
data << r.rmrk_dsc
|
||||
data << r.data_crt_tm
|
||||
data << r.gujia_src_co
|
||||
data << r.gujia_vlgtwn_str_mng
|
||||
data << r.gujia_lgc_admndiv_cd
|
||||
data << r.gujia_lgc_adiv
|
||||
data << r.gujia_bds_bds_st
|
||||
map.put(r.gujia_comm_id, data)
|
||||
});
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def delPgData(){
|
||||
sqlInstance.execute("delete from base.bigdata_community")
|
||||
println "base.bigdata_community 数据已删除"
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_community values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
|
||||
ps.toString()
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
def saveToPgSql2(rows){
|
||||
def newRows = []
|
||||
for(def r : rows) {
|
||||
def newR = []
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
// r[i] = null
|
||||
newR << null
|
||||
}else{
|
||||
newR << r[i]
|
||||
}
|
||||
}
|
||||
newR << null
|
||||
newR << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
newR << null
|
||||
newR << '2999-12-31'
|
||||
newR << null
|
||||
newR << null
|
||||
newR << null
|
||||
|
||||
newRows << newR
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
def index = 0
|
||||
sqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : newRows) {
|
||||
index++
|
||||
try{
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
|
||||
index
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split("\\|@\\|")
|
||||
rows << row
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
def updateVersion(def time){
|
||||
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
|
||||
for(def tableName : tables){
|
||||
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
|
||||
sqlInstance.executeUpdate(sql)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+786
@@ -0,0 +1,786 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 成交数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -24, "yyyyMM"));
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
|
||||
|
||||
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
|
||||
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
|
||||
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataDealBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = ["乌鲁木齐"] //CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
|
||||
def rows = []
|
||||
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def ppeDists = [:]
|
||||
|
||||
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
|
||||
println "历史id加载量:"+dealIds.size()
|
||||
def statMap = [:]//getDistrictStatByCity(city, weekend)
|
||||
if(city=="上海"){
|
||||
def youDataTrade = getYouDaData(statMap)
|
||||
def youdaDatas = createData(city, youDataTrade, cityCodeDataMap, distCodeMap, "youda", dealIds)
|
||||
rows.addAll(youdaDatas)
|
||||
println city +" "+youdaDatas.size()
|
||||
}
|
||||
|
||||
def trades = getCityData(city, statMap, ppeDists)
|
||||
if(trades){
|
||||
def rets = createData(city, trades, cityCodeDataMap, distCodeMap, "other", dealIds)
|
||||
rows.addAll(rets)
|
||||
println "【${index}】"+city +" 成交共:"+rets.size()
|
||||
|
||||
def cityDatas = []
|
||||
cityDatas << headers
|
||||
cityDatas.addAll(rows)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
saveToPgSql(rows)
|
||||
|
||||
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
|
||||
// saveToPgSql2(datas)
|
||||
}
|
||||
|
||||
def readTxt(def fileName){
|
||||
File file = new File(fileName);
|
||||
def rows = [];
|
||||
|
||||
file.eachLine{
|
||||
def row = it.split("\\|@\\|")
|
||||
rows << row
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
|
||||
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
dealIds = Collections.synchronizedList(dealIds);
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
|
||||
// def lianjiaPriceMap = findThirdPartyPrice("上海")
|
||||
// def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd')
|
||||
// def lianjiaPriceMap = getDistrictStatByCity("上海", startDate)
|
||||
|
||||
def ccbPriceMap = null
|
||||
if(city == "上海"){
|
||||
ccbPriceMap = getCommunityPrices("上海")
|
||||
}
|
||||
|
||||
for(def i=0;i<rets.size();i++){
|
||||
final def trade = rets[i]
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
if(dealIds.contains(trade._id.toString())){
|
||||
println "重复:"+trade._id.toString() + " "+ trade.city+ " " +trade.district_name
|
||||
}else{
|
||||
if(source == "youda"){
|
||||
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
dealIds << trade._id.toString()
|
||||
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
|
||||
}
|
||||
}else{
|
||||
def ret = converData(trade, cityCodeDatas, distCodeMap)
|
||||
if(ret){
|
||||
result << ret
|
||||
dealIds << trade._id.toString()
|
||||
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await(5, TimeUnit.MINUTES);
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city) {
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte', endDate))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for (def db : dbcursor) {
|
||||
def price = null
|
||||
if (db.lianjia_esf_price && db.lianjia_esf_price > 3000) {
|
||||
price = db.lianjia_esf_price
|
||||
}
|
||||
if (!price && db.beike_esf_price && db.beike_esf_price > 3000) {
|
||||
price = db.beike_esf_price
|
||||
}
|
||||
if(price){
|
||||
map.put(db.region+"##"+db.name, price)
|
||||
}
|
||||
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap, def ccbPriceMap){
|
||||
def ret = []
|
||||
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
|
||||
if(!obj){
|
||||
return ret;
|
||||
}
|
||||
def price = ccbPriceMap.get(obj._id.toString())
|
||||
if(price){
|
||||
def mod = java.lang.Math.abs((trade.price-price)*100/price)
|
||||
if(mod<=20){
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << trade.city
|
||||
def data = distCodeMap.get(trade.fjRegion)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
def street = ""
|
||||
def streetNumber = ""
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}
|
||||
if(!street){
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
if(jsonObj.searchAddress.street && jsonObj.searchAddress.street.contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(!street){
|
||||
if(obj.address){
|
||||
if(obj.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(obj.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
street = obj.address.toString().substring(0, obj.address.toString().indexOf(number))
|
||||
streetNumber = number
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(obj.city, obj.address, "property", false, trade.region, obj.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
street = jsonObj.searchAddress.street
|
||||
streetNumber = jsonObj.searchAddress.streetNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret << street
|
||||
ret << streetNumber
|
||||
|
||||
|
||||
if(obj){
|
||||
ret << obj._id.toString()
|
||||
ret << trade.fjName
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(obj.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(obj.property_life)
|
||||
ret << ""//期数
|
||||
String roomNum = DataUtil.match(trade.address, Pattern.compile("\\d+室"))
|
||||
if(roomNum && roomNum.contains("0")){
|
||||
ret << roomNum.substring(0, roomNum.lastIndexOf("0"))
|
||||
}else{
|
||||
ret << trade.floor
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
def arrs = CcbCodeUtils.parseBuildType(obj.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(obj.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(obj.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.price
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "99"
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.totalPrice * 10000
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
def converData(def trade, def cityCodeDatas, def distCodeMap){
|
||||
def ret = []
|
||||
def data = distCodeMap.get(trade.region)
|
||||
if(!data){
|
||||
println trade.city +" "+trade.region+" 没有找到对应的区域"
|
||||
return ret
|
||||
}
|
||||
ret << cityCodeDatas.get(trade.city)
|
||||
ret << trade._id.toString()
|
||||
ret << CcbCommonUtils.getCcbCity(trade.city)
|
||||
if(data){
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(trade.address){
|
||||
if(trade.address.toString().contains("弄")){
|
||||
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
|
||||
if(number){
|
||||
ret << trade.address.toString().substring(0, trade.address.toString().indexOf(number))
|
||||
ret << number
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}else{
|
||||
def jsonObj = OpenApiClient.addressMatch(trade.city, trade.address, "property", false, trade.region, trade.name)
|
||||
if(jsonObj && jsonObj.searchAddress){
|
||||
ret << jsonObj.searchAddress.street
|
||||
ret << jsonObj.searchAddress.streetNumber
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
ret << ""
|
||||
}
|
||||
ret << trade.dist_id.toString()
|
||||
ret << trade.master_name
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.parseHouseProperty(trade.house_roperty)//权属
|
||||
ret << CcbCodeUtils.parseHousePropertyYear(trade.property_life)
|
||||
ret << ""//期数
|
||||
ret << trade.floor
|
||||
ret << ""
|
||||
ret << trade.height
|
||||
// ret << CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
def arrs = CcbCodeUtils.parseBuildType(trade.build_type)//建筑类型
|
||||
def buildType = ''
|
||||
if(arrs){
|
||||
buildType = arrs[0]
|
||||
}
|
||||
ret << buildType
|
||||
ret << CcbCodeUtils.getBuildStructureCode(trade.build_structure)
|
||||
ret << trade.area
|
||||
ret << ""
|
||||
ret << getHouseRoom(buildType, trade.room, trade.hall)
|
||||
ret << trade.room
|
||||
ret << trade.hall
|
||||
ret << ""
|
||||
ret << trade.toilet
|
||||
ret << CcbCodeUtils.parseHouseUse(trade.estate_type2)
|
||||
ret << ""
|
||||
ret << CcbCodeUtils.getDirectionCode(trade.direction)
|
||||
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
|
||||
ret << ""
|
||||
ret << trade.avg_price
|
||||
ret << trade.total_price
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
|
||||
ret << "02"
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
def root = sourceMap.get(trade.source)
|
||||
if(root){
|
||||
if(CcbCommonUtils.rootMap.get(root)){
|
||||
ret << CcbCommonUtils.rootMap.get(root)
|
||||
}else{
|
||||
ret << "99"
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(trade.create_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
ret << trade.total_price
|
||||
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getHouseRoom(buildtype, def roomcnt, def livingroomcnt){
|
||||
def housetype = null
|
||||
if(buildtype && ["0112","0110","0109"].contains(buildtype)){
|
||||
housetype = "07"
|
||||
}else if(buildtype && ["0107","0106","0105"].contains(buildtype)){
|
||||
housetype = "16"
|
||||
}else{
|
||||
if(roomcnt && roomcnt==4){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "14"
|
||||
}else{
|
||||
housetype = "11"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==3){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "13"
|
||||
}else{
|
||||
housetype = "10"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==2){
|
||||
if(livingroomcnt>1){
|
||||
housetype = "12"
|
||||
}else{
|
||||
housetype = "09"
|
||||
}
|
||||
}else if(roomcnt && roomcnt==1){
|
||||
housetype = "08"
|
||||
}else{
|
||||
housetype="99"
|
||||
}
|
||||
}
|
||||
return housetype;
|
||||
}
|
||||
|
||||
def getCityData(def city, def statMap, def ppeDistsMap){
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"], "all")
|
||||
println city +" 小区量:"+dists.size()
|
||||
def tradesList = Collections.synchronizedList(new ArrayList());
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def price = statMap.get(dist.region+"@"+dist.name)
|
||||
def startPrice = null
|
||||
def endPrice = null
|
||||
// if(price){
|
||||
// if(dist.city=="上海"){
|
||||
// startPrice = (int)(price - price*0.1)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }else if(dist.city=="杭州"){
|
||||
// startPrice = (int)(price - price*0.15)
|
||||
// endPrice = (int)(price*0.15 + price)
|
||||
// }else{
|
||||
// startPrice = (int)(price - price*0.5)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// }
|
||||
// def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
|
||||
// if(!trades && dist.address){
|
||||
// def flag = dist.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
// if(flag){
|
||||
// trades = CcbCommonUtils.findTradesDetailByAddress(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
|
||||
//
|
||||
// if(!trades){
|
||||
// trades = CcbCommonUtils.findTradesDetailByAddressAlias(dist, startDate, endDate, startPrice, endPrice)//地址是小区名
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// }
|
||||
def trades = CcbCommonUtils.findTradesDetailById(dist, startDate, endDate)
|
||||
if(trades){
|
||||
// def unitPrice= ppeDistsMap.get(dist._id.toString())
|
||||
// if(unitPrice){
|
||||
// for(def t : trades){
|
||||
// t.avg_price = (int)(t.avg_price*0.03+t.avg_price)
|
||||
// }
|
||||
// }
|
||||
tradesList.addAll(trades)
|
||||
}
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return tradesList
|
||||
}
|
||||
|
||||
def getYouDaData(statMap){
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def q = new BasicDBObject()
|
||||
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lte', endDate));
|
||||
q.put("area", new BasicDBObject('$gte', 20))
|
||||
// q.put("status", 0)
|
||||
def dbcursor = youda_esfhouse_coll.find(q).batchSize(2000)
|
||||
|
||||
def rets = []
|
||||
def index = 0
|
||||
for(def obj : dbcursor){
|
||||
|
||||
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房","其它"].contains(obj.houseType)){
|
||||
rets << obj;
|
||||
// def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
|
||||
// if(price){
|
||||
// def startPrice = 0
|
||||
// def endPrice= 0
|
||||
// if(obj.fjCity=="上海"){
|
||||
// startPrice = (int)(price - price*0.1)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }else{
|
||||
// startPrice = (int)(price - price*0.5)
|
||||
// endPrice = (int)(price*0.5 + price)
|
||||
// }
|
||||
// if(obj.price>=startPrice && obj.price<=endPrice){
|
||||
// rets << obj;
|
||||
// }else{
|
||||
// index++
|
||||
// println index + " "+obj
|
||||
// }
|
||||
// }
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", new BasicDBObject('$gte', weekend))
|
||||
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("weekend",-1))
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
def price = map.get(d.region+"##"+d.district_name)
|
||||
if(!price && d.value_type==2){
|
||||
map.put(d.region+"##"+d.district_name, d.avg_price)
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findHistoryDealIds(def city){
|
||||
def sql = "SELECT gujia_bds_mdl_id from base.bigdata_deal where urbn_nm='${city}'"
|
||||
def ids = []
|
||||
try{
|
||||
pgsqlInstance.eachRow(sql,{
|
||||
ids << it.gujia_bds_mdl_id
|
||||
});
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
|
||||
return ids
|
||||
}
|
||||
|
||||
|
||||
|
||||
def addDealPriceMatch(city){
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def startDate = DateUtils.parseFromTightByPattern(startDate.toString()+"01", "yyyyMMdd")
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210101", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20221001", "yyyyMMdd"))
|
||||
def trades = CcbCommonUtils.findTradesByMonth(city, startDate)
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(trades.size())
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
for(def i=0;i<trades.size();i++){
|
||||
final def t = trades[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
count_.addAndGet(1)
|
||||
try{
|
||||
if(!t.match_id){
|
||||
def region = RegionData.formatRegion(t.city, t.region)
|
||||
def rMap = null
|
||||
if(region) {
|
||||
if(t.house_use && t.house_use.contains("商住")){
|
||||
if(!t.name.contains("商住")){
|
||||
t.name = t.name +"商住楼"
|
||||
}
|
||||
}
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, t.name, null);
|
||||
if(!rMap && t.address){
|
||||
def flag = t.address.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')
|
||||
if(flag){
|
||||
rMap = SeaweedDataMatchProcess.doMatch(t.city, region, null, t.address);
|
||||
}
|
||||
}
|
||||
if(rMap && rMap.region==region){
|
||||
CcbCommonUtils.updateDealPrice(t._id, rMap.city, rMap.region, rMap.name, rMap.id)
|
||||
}else {
|
||||
if (rMap) {
|
||||
println "区域不同"
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
println count_.get()+" "+t+" "+ rMap
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def delPgData(){
|
||||
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
pgsqlInstance.execute("delete from base.bigdata_deal where p9_start_date='${month}'")
|
||||
println "base.bigdata_deal 数据已删除"
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_deal values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
|
||||
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
|
||||
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql2(rows){
|
||||
def newRows = []
|
||||
for(def r : rows) {
|
||||
def newR = []
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
// r[i] = null
|
||||
newR << null
|
||||
}else{
|
||||
newR << r[i]
|
||||
}
|
||||
}
|
||||
newR << null
|
||||
newR << "2023-09-28"
|
||||
newR << null
|
||||
newR << '2999-12-31'
|
||||
newR << null
|
||||
newR << null
|
||||
newR << null
|
||||
|
||||
newRows << newR
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : newRows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
|
||||
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
|
||||
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
+968
@@ -0,0 +1,968 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.ccbevaluation.CcbCommonUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.google.common.hash.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 出售房源数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
static def DIRECTIONS = ["东北", "西北", "东南", "西南", "南北", "东西", "东", "南", "西", "北"]
|
||||
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
|
||||
static{
|
||||
FITMENTS_TYPES_MAPS.put("毛坯", ["毛坯", "清水", "无装修", "独栋毛坯", "毛胚", "白坯", "旧装", "坯", "原房", "毛坏"])
|
||||
FITMENTS_TYPES_MAPS.put("简装", ["简装", "普通装修", "普通", "简单装修", "简易装修", "普装", "一般装修", "一般", "简", "粗装修"])
|
||||
FITMENTS_TYPES_MAPS.put("精装", ["精装", "全装修", "全装", "中等装修", "中装", "中档装修", "精致装修", "中档", "中等", "精", "中修", "优质装修", "时尚装修", "良装", "中高挡"])
|
||||
FITMENTS_TYPES_MAPS.put("豪装", ["豪华装修", "高档", "豪华", "豪装", "高级装修", "高装", "奢侈"])
|
||||
}
|
||||
|
||||
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
|
||||
|
||||
public static ExecutorService fixedThreadPool2 = Executors.newFixedThreadPool(200)
|
||||
|
||||
|
||||
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwBigdataForsaleBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
//目前查询库中是否已存在,用的是Oracle
|
||||
def index = 0
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
|
||||
def startDate = lastMonth+"18"
|
||||
def endDate = curMonth+"19" //正式运行时curMonth要改
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
|
||||
def houseIdsfilter = findAllHouseIdsByCity()
|
||||
|
||||
for(def city : citys){
|
||||
index++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def statCity(def citys){
|
||||
def datas = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\无房源城市.xlsx",0,1)
|
||||
def houseCitys = []
|
||||
for(def d : datas){
|
||||
houseCitys << d['C']
|
||||
}
|
||||
for(def city : citys){
|
||||
if(!houseCitys.contains(city)){
|
||||
def scity = CcbCommonUtils.specialCityMap.get(city)
|
||||
if(!scity){
|
||||
println city
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
|
||||
|
||||
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
|
||||
println city + "(原)房源总量:"+retMap.size()
|
||||
|
||||
def allDatas = retMap.values()
|
||||
saveToPgSql(allDatas)
|
||||
}
|
||||
|
||||
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println city +" 小区量:"+districts.size()
|
||||
|
||||
|
||||
BloomFilter<CharSequence> curhousefilter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
def retMap = new ConcurrentHashMap()
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def rMap = districts[i];
|
||||
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def commId = rMap.get("_id").toString()
|
||||
|
||||
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId, cityCodeDataMap)
|
||||
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
|
||||
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
|
||||
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
||||
// }
|
||||
if(ret){
|
||||
def areaData = distCodeMap.get(rMap.region)
|
||||
if(areaData){
|
||||
def regionCode = areaData.district_code
|
||||
def cityCode = cityCodeDataMap.get(city)
|
||||
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
|
||||
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
|
||||
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
|
||||
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
|
||||
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
|
||||
for(def m : ret){
|
||||
if(!houseIdsfilter.mightContain(m.houseId) && !curhousefilter.mightContain(m.houseId)){
|
||||
m.city_code = cityCode
|
||||
m.region = areaData.district
|
||||
m.region_code = regionCode
|
||||
m.district_id = commId
|
||||
m.district_name = rMap.name
|
||||
m.build_year = buildYear;
|
||||
m.proptype = housePro
|
||||
m.propyears = propertyYear;
|
||||
m.estate_type = estateType
|
||||
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
|
||||
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
|
||||
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
|
||||
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
|
||||
m.address = rMap.address
|
||||
m.build_structure_code = struCode
|
||||
m.house_structure_code = null//getHouseStructureCode();
|
||||
m.source = "FJW"
|
||||
retMap.put(m.houseId, converData(m))
|
||||
curhousefilter.put(m.houseId)
|
||||
houseIdsfilter.put(m.houseId)
|
||||
}else{
|
||||
println city +" "+m.houseId+" 已存在"
|
||||
}
|
||||
}
|
||||
}else{
|
||||
println city +" "+rMap.region+" 没有找到对应的区域"
|
||||
}
|
||||
|
||||
}
|
||||
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size()
|
||||
|
||||
} catch (Exception e) {
|
||||
println totalCount.get() +" 查房源Error: "+city+" "+rMap.region+" "+rMap.name+" "+e.getMessage()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return retMap
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def converData(def m){
|
||||
def data = []
|
||||
data << m.city_code
|
||||
data << m.houseId
|
||||
data << ""
|
||||
data << m.city
|
||||
data << m.region_code
|
||||
data << m.region
|
||||
data << m.district_id
|
||||
data << m.district_name
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.address==null?"":m.address==null
|
||||
data << m.build_year==null?"":m.build_year
|
||||
data << m.proptype==null?"":m.proptype
|
||||
data << m.propyears==null?"":m.propyears
|
||||
data << m.avg_price==null?"":m.avg_price
|
||||
data << m.total_price==null?"":m.total_price
|
||||
data << ""
|
||||
data << m.floor==null?"":m.floor
|
||||
data << m.height==null?"":m.height
|
||||
data << m.estate_type==null?"":m.estate_type
|
||||
data << m.room_structure_code==null?"":m.room_structure_code
|
||||
data << m.build_structure_code==null?"":m.build_structure_code
|
||||
data << m.area==null?"":m.area
|
||||
data << ""
|
||||
data << m.house_type==null?"":m.house_type
|
||||
data << m.room==null?"":m.room
|
||||
data << m.hall==null?"":m.hall
|
||||
data << ""
|
||||
data << m.toilet==null?"":m.toilet
|
||||
data << m.direction_code==null?"":m.direction_code
|
||||
data << m.fitment_code==null?"":m.fitment_code
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.p_date==null?"":m.p_date
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.root
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << ""
|
||||
data << m.c_date==null?"":m.c_date
|
||||
data << m.m_date==null?"":m.m_date
|
||||
data << m.source
|
||||
data << ""
|
||||
return data
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId, def cityCodeDataMap){
|
||||
|
||||
def rets1 = []//findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets2 = findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
rets1.addAll(rets2)
|
||||
// if(rets1){
|
||||
// def newRets = []
|
||||
// def cityCode = cityCodeDataMap.get(city)
|
||||
// def houseIds = getAllHouseIds(commId, cityCode)
|
||||
// for(def r : rets1){
|
||||
// if(!houseIds.contains(r.houseId)){
|
||||
// newRets << r
|
||||
// }else{
|
||||
// println r.houseId +" 已存在"
|
||||
// }
|
||||
// }
|
||||
// return newRets
|
||||
// }
|
||||
// return [];
|
||||
return rets1;
|
||||
}
|
||||
|
||||
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
// if(seawater.height && seawater.height<1){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = CcbCommonUtils.getCcbCity(city)
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(seawater.p_date<=20200101){
|
||||
//seawater.p_date.toString().replace("2019","2020")//
|
||||
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
|
||||
// if(houseIds.contains(seawater.id)){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
// def count = isExistFromHouse(seawater.id)
|
||||
// if(count){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
println name +" "+houses.size()+" last:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
|
||||
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
|
||||
|
||||
//用以下方法会查出发布时间很久以前的
|
||||
// QueryBuilder dateQueryBuilder = QueryBuilders.boolQuery()
|
||||
// dateQueryBuilder.should(date_range)
|
||||
// dateQueryBuilder.should(date_range_mdate)
|
||||
// QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
// .must(area_range).must(dateQueryBuilder)
|
||||
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
|
||||
|
||||
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
|
||||
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
}
|
||||
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
|
||||
def ids = new HashSet()
|
||||
for(def h : rets1){
|
||||
ids << h.houseId
|
||||
}
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
DBObject seawater = new BasicDBObject(map)
|
||||
if(ids && ids.contains(seawater.id)){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(estate_type2 && estate_type2=="普通住宅"){
|
||||
if(seawater.estate_type && (seawater.estate_type.contains("商住") || seawater.estate_type.contains("商业") || seawater.estate_type.contains("商务"))){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
if(seawater.estate_type&& seawater.estate_type.contains("别墅")){
|
||||
println city +" "+name+" "+seawater._id.toString()+" "+seawater.estate_type
|
||||
continue;
|
||||
}
|
||||
|
||||
def vallaFlag = isVilla(seawater.area, seawater.name, seawater.height, seawater.room)
|
||||
if(vallaFlag){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(!seawater.total_price){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.avg_price){
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!seawater.height){
|
||||
seawater.height = null
|
||||
}
|
||||
// if(seawater.height && seawater.height<1){
|
||||
// continue;
|
||||
// }
|
||||
if(seawater.height && seawater.height>60){
|
||||
continue;
|
||||
}
|
||||
|
||||
// if(!seawater.room){
|
||||
// continue;
|
||||
// }
|
||||
if(!seawater.room){
|
||||
seawater.room = null
|
||||
}
|
||||
|
||||
if(seawater.room){
|
||||
if(seawater.room>=11 || seawater.room<1){
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if(!seawater.hall){
|
||||
seawater.hall = null
|
||||
}
|
||||
if(seawater.hall && (seawater.hall>=5 || seawater.hall<1)){
|
||||
continue;
|
||||
}
|
||||
if(!seawater.toilet){
|
||||
seawater.toilet = null
|
||||
}
|
||||
if(seawater.toilet && (seawater.toilet>=5 || seawater.toilet<1)){
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
def floor = parseFloor(seawater.floor_info)
|
||||
if(seawater.height && floor && floor.isNumber()){
|
||||
if(floor.toInteger()>seawater.height){
|
||||
floor = null
|
||||
seawater.height = null
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if(!floor){
|
||||
seawater.floor = null
|
||||
}
|
||||
if(floor && floor.toInteger()<1){
|
||||
floor = null
|
||||
}
|
||||
|
||||
def rowMap = [:];
|
||||
if(cat == "rent"){
|
||||
rowMap.rent_type = seawater.rent_type
|
||||
if(seawater.avg_price && seawater.avg_price>1000){
|
||||
continue
|
||||
}
|
||||
|
||||
if(seawater.avg_price && seawater.avg_price<10){
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
rowMap.houseId = seawater.id
|
||||
rowMap.city = CcbCommonUtils.getCcbCity(city)
|
||||
//rowMap.region =seawater.region
|
||||
//rowMap.address = seawater.address
|
||||
|
||||
|
||||
rowMap.area = seawater.area
|
||||
rowMap.total_price = seawater.total_price
|
||||
rowMap.avg_price = seawater.avg_price
|
||||
rowMap.height = seawater.height=="null"?null:seawater.height
|
||||
if(rowMap.height && rowMap.height<1){
|
||||
rowMap.height = null
|
||||
}
|
||||
rowMap.floor = floor=="null"?null:floor
|
||||
rowMap.room = seawater.room=="null"?null:seawater.room
|
||||
rowMap.hall = seawater.hall=="null"?null:seawater.hall
|
||||
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
|
||||
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
|
||||
if(CcbCommonUtils.rootMap.get(seawater.root)){
|
||||
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
|
||||
}else{
|
||||
rowMap.root = "99"
|
||||
}
|
||||
|
||||
def direction = parseDirection(seawater.direction)
|
||||
rowMap.direction = direction=="null"?null:direction
|
||||
|
||||
def fitment = parseFitment(seawater.fitment)
|
||||
rowMap.fitment = fitment=="null"?null:fitment
|
||||
|
||||
if(seawater.p_date instanceof String){
|
||||
seawater.p_date = seawater.p_date.toInteger()
|
||||
}
|
||||
if(seawater.p_date<=20200101){
|
||||
//seawater.p_date.toString().replace("2019","2020")//
|
||||
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
|
||||
rowMap.c_date = rowMap.p_date
|
||||
rowMap.m_date = rowMap.p_date
|
||||
}else{
|
||||
rowMap.p_date = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(seawater.p_date.toString()), "yyyy-MM-dd HH:mm:ss")
|
||||
rowMap.c_date = DateUtils.getConverDate(seawater.m_date)
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
rets << rowMap
|
||||
|
||||
// if(houseIds.contains(seawater.id)){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
// def count = isExistFromHouse(seawater.id)
|
||||
// if(count){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
println name +" "+houses.size()+" last:"+rets.size()
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHousesByDistrict(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) {
|
||||
int retryCount = 0;
|
||||
boolean success = false;
|
||||
def rets = []
|
||||
|
||||
while (retryCount < MAX_TRY_COUNT && !success) {
|
||||
try {
|
||||
rets = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
// 如果操作成功,设置success为true
|
||||
success = true;
|
||||
} catch (Exception e) {
|
||||
retryCount++;
|
||||
println "查询重试 " + retryCount + "/" + MAX_TRY_COUNT
|
||||
// 这里可以选择在重试之前等待一段时间
|
||||
Thread.sleep(3000); // 例如等待1秒
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) {
|
||||
println "重试5次后依然失败: " + MAX_TRY_COUNT + " "+city +" "+region+" "+name
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 是否是别墅
|
||||
* @return
|
||||
*/
|
||||
public boolean isVilla(area, name, height, room) {
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
boolean ret = false;
|
||||
if (height != null && room != null) {
|
||||
if (height <= 3) {
|
||||
if (area > 200 && room >= 5) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
} else if (height == null && room != null) {
|
||||
if (area > 360 && room >= 6) {
|
||||
ret = true;
|
||||
}
|
||||
} else if (height == null && room == null) {
|
||||
if (area > 500) {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
def getPublicDate(def pDate){
|
||||
|
||||
if(pDate>=20181001 && pDate<20200201){
|
||||
return DateUtils.getRandomDate("2018-10-01", "2020-02-01")
|
||||
}
|
||||
def pdate = DateUtils.formatDateByPattern(DateUtils.parseFromTightYYYYMMDD2(pDate), "yyyy-MM-dd HH:mm:ss")
|
||||
return pdate;
|
||||
}
|
||||
|
||||
static def parseFloor(def floorDesc){
|
||||
if(floorDesc){
|
||||
floorDesc = floorDesc.replaceAll("[\\s ]+", "")
|
||||
if(floorDesc){
|
||||
Pattern pat1 = Pattern.compile("第(\\d{1,2})层");
|
||||
Matcher matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
pat1 = Pattern.compile("(\\d{1,2})层?(/|\\()共?\\d{1,2}(层|F)?");
|
||||
matcher1 = pat1.matcher(floorDesc);
|
||||
if (matcher1.find()) {
|
||||
return matcher1.group(1)
|
||||
}
|
||||
// if(floorDesc.contains("中楼层") || floorDesc.contains("中层")){
|
||||
// return "中层"
|
||||
// }
|
||||
//
|
||||
// if(floorDesc.contains("高楼层") || floorDesc.contains("高层")){
|
||||
// return "高层"
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(floorDesc.contains("低楼层") || floorDesc.contains("低层")){
|
||||
// return "低层"
|
||||
// }
|
||||
if(floorDesc.isNumber()){
|
||||
return floorDesc;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private String csvHandlerStr(String str) {
|
||||
//csv格式如果有逗号,整体用双引号括起来;如果里面还有双引号就替换成两个双引号,这样导出来的格式就不会有问题了
|
||||
String tempDescription=str;
|
||||
//如果有逗号
|
||||
if(str.contains(",")||str.contains(",")){
|
||||
//如果还有双引号,先将双引号转义,避免两边加了双引号后转义错误
|
||||
if(str.contains("\"")){
|
||||
// tempDescription=str.replace("\"", "\"\"");
|
||||
tempDescription=str.replace("\"", "");
|
||||
}
|
||||
//在将逗号转义
|
||||
tempDescription="\""+tempDescription+"\"";
|
||||
}
|
||||
return tempDescription;
|
||||
}
|
||||
|
||||
|
||||
static String parseFitment(String fitmentDesc) {
|
||||
if (fitmentDesc == null) {
|
||||
return null
|
||||
}
|
||||
fitmentDesc = fitmentDesc.replaceAll("[\\s ]+", "")
|
||||
if (fitmentDesc.equals("非毛坯")) {
|
||||
return null
|
||||
}
|
||||
Iterator<Map.Entry<String, List<String>>> it = FITMENTS_TYPES_MAPS
|
||||
.entrySet().iterator()
|
||||
Map.Entry<String, List<String>> entry
|
||||
while (it.hasNext()) {
|
||||
entry = it.next()
|
||||
if (fitmentDesc.contains(entry.getKey())) {
|
||||
return entry.getKey()
|
||||
}
|
||||
for (String fitmentKey : entry.getValue()) {
|
||||
if (fitmentDesc.contains(fitmentKey)) {
|
||||
return entry.getKey()
|
||||
}
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
static String parseDirection(String directionDesc) {
|
||||
if (directionDesc == null) {
|
||||
return null
|
||||
}
|
||||
directionDesc = directionDesc.replaceAll("(\\(|(|《|\"|“|【|〔).+?(\\)|》|)|\"|”|】|〕)", "#_#")
|
||||
StringBuffer sb = new StringBuffer()
|
||||
for (int i = 0; i < directionDesc.length(); i++) {
|
||||
String c = directionDesc.substring(i, i + 1)
|
||||
if (DIRECTIONS.contains(c)) {
|
||||
if (directionDesc.substring(i).startsWith(c + "#_#")) {
|
||||
sb.append(c + " ")
|
||||
} else {
|
||||
sb.append(c)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().startsWith(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
for (String d : DIRECTIONS) {
|
||||
if (sb.toString().contains(d)) {
|
||||
return d
|
||||
}
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def marge(){
|
||||
File file = new File("D:\\数据\\建行\\新建文本文档.txt");
|
||||
def ids = new HashSet();
|
||||
def index = 0
|
||||
|
||||
file.eachLine{it->
|
||||
index++
|
||||
if(index>1){
|
||||
def arrs = it.split("\\|@\\|")
|
||||
def id = arrs[1]
|
||||
if(!ids.contains(id)){
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
}
|
||||
println index
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
def getAllHouseIds(def commid, def cityCode){
|
||||
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"' and commid = '"+commid+"'";
|
||||
|
||||
def houseIds = new HashSet()
|
||||
sqlInstance.eachRow(sql,{it->
|
||||
houseIds << it.FORSALEID
|
||||
});
|
||||
return houseIds;
|
||||
}
|
||||
|
||||
def findAllHouseIdsByCity(){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
// def houseIds = new HashSet()
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
|
||||
filter.put(it.gujia_sell_id)
|
||||
});
|
||||
|
||||
// pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale_2021",{it->
|
||||
// filter.put(it.gujia_sell_id)
|
||||
// });
|
||||
return filter;
|
||||
}
|
||||
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_forsale values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40],
|
||||
r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58])
|
||||
ps.toString()
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
}
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.CcbCodeUtils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import org.bson.types.ObjectId
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 人工指导价同步
|
||||
*/
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol();
|
||||
|
||||
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwGuiDingPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01")
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
}
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def findFangDatas(def districts){
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
println " 小区量:"+districts.size()
|
||||
def result = Collections.synchronizedList(new ArrayList());
|
||||
for(def i=0;i<districts.size();i++){
|
||||
final def dist = districts[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def obj = findMatchThirdPrice(dist)
|
||||
if(obj && obj.avg_price){
|
||||
result << ["dist_id":dist._id.toString(),"city":dist.city, "region":dist.region, "name":dist.name, "avg_price":obj.avg_price,"row_id":obj._id.toString(), "m_date":obj.m_date]
|
||||
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name +" "+obj.avg_price
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return result
|
||||
}
|
||||
|
||||
def findPersonDatas(def districts, def dataMap){
|
||||
def result = []
|
||||
for(def i=0;i<districts.size();i++) {
|
||||
def dist = districts[i];
|
||||
def price = dataMap.get(dist._id.toString())
|
||||
if (price) {
|
||||
result << ["dist_id": dist._id.toString(), "city": dist.city, "region": dist.region, "name": dist.name, "avg_price": price.toInteger(), "row_id": new ObjectId(), "m_date": new Date()]
|
||||
println dist.city + " " + dist.region + " " + dist.name + " " + price
|
||||
}
|
||||
}
|
||||
return result;
|
||||
|
||||
}
|
||||
|
||||
def findMatchThirdPrice(dist){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", dist.city)
|
||||
q.put("match_region", dist.region)
|
||||
q.put("match_name", dist.name)
|
||||
q.put("s_date", sDate)
|
||||
q.put("source","搜房")
|
||||
def obj = dataProcessColl.findOne(q)
|
||||
return obj
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
if(areaData){
|
||||
ret << ps._id.toString()
|
||||
ret << dist._id.toString()
|
||||
ret << dist.name
|
||||
ret << cityCodeDatas.get(dist.city)
|
||||
ret << CcbCommonUtils.getCcbCity(dist.city)
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.guiding_value
|
||||
ret << "房价网"
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createDatas(cityCodeDatas, distCodeMap, datas){
|
||||
def rets = []
|
||||
if(datas) {
|
||||
for (def ps : datas) {
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(ps.region)
|
||||
if (areaData) {
|
||||
ret << ps.row_id
|
||||
ret << ps.dist_id
|
||||
ret << ps.name
|
||||
ret << cityCodeDatas.get(ps.city)
|
||||
ret << ps.city
|
||||
ret << areaData.district_code
|
||||
ret << areaData.district
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ps.avg_price
|
||||
ret << "房价网"
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(ps.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createPersonDatas(cityCodeDatas, distCodeMap, dataMap){
|
||||
|
||||
}
|
||||
|
||||
def getDistrictMap(def city){
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat", "alias"], "all");
|
||||
def distMap = [:]
|
||||
for(def d : districts){
|
||||
distMap.put(d.region+"##"+d.name, d)
|
||||
}
|
||||
return distMap;
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.guided_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(300, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import groovy.sql.Sql
|
||||
|
||||
/**
|
||||
* ·¿¼ÛÍøÄÚ²¿Ê¹ÓÃ
|
||||
*/
|
||||
class FjwPgsqlCommUtils {
|
||||
|
||||
public static def database = "fangjia"
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
public static def findAllDistrictIdsByOracle(def cityid){
|
||||
def sql = "select gujia_comm_id from base.bigdata_community where urbn_id='${cityid}' and gujia_bds_bds_st=0";
|
||||
def set = new HashSet()
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
set << r.gujia_comm_id
|
||||
});
|
||||
return set
|
||||
}
|
||||
|
||||
public static def findDistGovPrice(def city){
|
||||
def ids = []
|
||||
sqlInstance.eachRow("SELECT gujia_bds_id from base.community_guide_price where urbn_nm='${city}'",{it->
|
||||
ids << it.gujia_bds_id
|
||||
});
|
||||
return ids;
|
||||
}
|
||||
|
||||
|
||||
public static def deleteGuidedPrice(def dealdate){
|
||||
def sql2 = "delete from base.guided_price where gujia_drc_dt='${dealdate}'"
|
||||
sqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
+675
@@ -0,0 +1,675 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.ccb.FileUtils
|
||||
import com.fangjia.script.gj.fjwPriceBaseData.FjwPgsqlCommUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
|
||||
/**
|
||||
* 建行模型估值项目
|
||||
* 第三方均价数据同步
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
|
||||
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304//
|
||||
def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302
|
||||
def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304
|
||||
// def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
|
||||
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new FjwThirdPartyPriceBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def stop = false
|
||||
def month = -1
|
||||
while(!stop){
|
||||
if(sDate<202004){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
println weekend.toString() +" "+sDate
|
||||
runDatas(citys)
|
||||
}
|
||||
month = month -1
|
||||
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
|
||||
if(sDate == 201912){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def runDatas(citys){
|
||||
def rows = []
|
||||
def index = 0
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
for(def city : citys){
|
||||
index++
|
||||
|
||||
def gocIdsPrices = []
|
||||
if(city == "深圳"){
|
||||
gocIdsPrices = FjwPgsqlCommUtils.findDistGovPrice(city)
|
||||
}
|
||||
|
||||
def statMap = getDistrictStatByCity(city, weekend)
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"], "all")
|
||||
def thirdPriceMap = findThirdPartyPrice(city, statMap)
|
||||
|
||||
def rets = createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocIdsPrices)
|
||||
rows.addAll(rets)
|
||||
println index+" "+city+" "+rets.size() +" "+rows.size()
|
||||
}
|
||||
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap){
|
||||
def rets = []
|
||||
for(def d : dists){
|
||||
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
if(data){
|
||||
def ret = []
|
||||
ret << third.id.toString()
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
ret << cityCodeDatas.get(d.city)
|
||||
ret << d.city
|
||||
ret << data.district_code
|
||||
ret << data.district
|
||||
ret << sDate
|
||||
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
ret << third.zhugezhaofang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
ret << third.anjuke_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
ret << third.soufang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.lianjia_esf_price){
|
||||
ret << third.lianjia_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.cityhouse_esf_price){
|
||||
ret << third.cityhouse_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.maitian_esf_price){
|
||||
ret << third.maitian_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.beike_esf_price){
|
||||
ret << third.beike_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.get("365house_esf_price")){
|
||||
ret << third.get("365house_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.zhongyuan_esf_price){
|
||||
ret << third.zhongyuan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.tuboshi_esf_price){
|
||||
ret << third.tuboshi_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
ret << third.qfang_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangtan_esf_price){
|
||||
ret << third.fangtan_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
if(third.fangduoduo_esf_price){
|
||||
ret << third.fangduoduo_esf_price
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
ret << third.get("5i5j_esf_price")
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def createCityDatas1(cityCodeDatas, distCodeMap, dists, thirdPriceMap, gocPrices){
|
||||
def allRows = []
|
||||
for(def d : dists){
|
||||
if(gocPrices && gocPrices.contains(d._id.toString())){
|
||||
continue;
|
||||
}
|
||||
def third = thirdPriceMap.get(d.region+"##"+d.name)
|
||||
if(third){
|
||||
def data = distCodeMap.get(d.region)
|
||||
if(data){
|
||||
def dataMap = [:]
|
||||
dataMap.COMMID = d._id.toString()
|
||||
dataMap.COMMNAME = d.name
|
||||
dataMap.CITYID = cityCodeDatas.get(d.city)
|
||||
dataMap.CITYNAME = d.city
|
||||
dataMap.DISTID = data.district_code
|
||||
dataMap.DISTNAME = data.district
|
||||
dataMap.YEAR_MONTH = sDate
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(third.zhugezhaofang_esf_price){
|
||||
price_map.zhuge_avg_price = third.zhugezhaofang_esf_price
|
||||
}
|
||||
if(third.anjuke_esf_price){
|
||||
price_map.anjuke_avg_price = third.anjuke_esf_price
|
||||
}
|
||||
if(third.soufang_esf_price){
|
||||
price_map.fang_avg_price = third.soufang_esf_price
|
||||
}
|
||||
if(third.lianjia_esf_price){
|
||||
price_map.lianjia_avg_price = third.lianjia_esf_price
|
||||
}
|
||||
if(third.cityhouse_esf_price){
|
||||
price_map.cityhouse_avg_price = third.cityhouse_esf_price
|
||||
}
|
||||
if(third.maitian_esf_price){
|
||||
price_map.maitian_avg_price = third.maitian_esf_price
|
||||
}
|
||||
if(third.beike_esf_price){
|
||||
price_map.ke_avg_price = third.beike_esf_price
|
||||
}
|
||||
if(third.get("365house_esf_price")){
|
||||
price_map.taofang_avg_price = third.get("365house_esf_price")
|
||||
}
|
||||
if(third.zhongyuan_esf_price){
|
||||
price_map.zhongyuan_avg_price = third.zhongyuan_esf_price
|
||||
}
|
||||
if(third.tuboshi_esf_price){
|
||||
price_map.tbs_avg_price = third.tuboshi_esf_price
|
||||
}
|
||||
if(third.qfang_esf_price){
|
||||
price_map.qfang_avg_price = third.qfang_esf_price
|
||||
}
|
||||
if(third.fangtan_esf_price){
|
||||
price_map.fangtan_avg_price = third.fangtan_esf_price
|
||||
}
|
||||
if(third.fangduoduo_esf_price){
|
||||
price_map.fangduoduo_avg_price = third.fangduoduo_esf_price
|
||||
}
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
def createCityDatas2(cityCodeDatas, distCodeMap, dists, thirdPriceMap) {
|
||||
def rets = []
|
||||
for (def d : dists) {
|
||||
|
||||
def third = thirdPriceMap.get(d.region + "##" + d.name)
|
||||
if (third) {
|
||||
def data = distCodeMap.get(d.region)
|
||||
if (data) {
|
||||
third.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << cityCodeDatas.get(d.city)
|
||||
ret << data.district_code
|
||||
ret << d._id.toString()
|
||||
ret << d.name
|
||||
def web = ""
|
||||
if(k=="zhugezhaofang_esf_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_esf_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="soufang_esf_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_esf_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_esf_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_esf_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="beike_esf_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="365house_esf_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_esf_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tuboshi_esf_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_esf_price"){
|
||||
ret << "Q房"
|
||||
}else if(k=="fangtan_esf_price"){
|
||||
ret << "房探"
|
||||
}else if(k=="fangduoduo_esf_price"){
|
||||
ret << "房多多"
|
||||
}else if(k=="5i5j_esf_price"){
|
||||
ret << "我爱我家"
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << sDate
|
||||
ret << v
|
||||
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << "FJW"
|
||||
rets << ret
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData3(cityid){
|
||||
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def dataMap = [:]
|
||||
|
||||
dataMap.COMMID = r.COMMID
|
||||
dataMap.COMMNAME = r.COMMNAME
|
||||
dataMap.CITYID = r.CITYID
|
||||
dataMap.CITYNAME = r.CITYNAME
|
||||
dataMap.DISTID = r.DISTID
|
||||
dataMap.DISTNAME = r.DISTNAME
|
||||
dataMap.YEAR_MONTH = r.YEAR_MONTH
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(r.ZHUGE_AVG_PRICE){
|
||||
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ANJUKE_AVG_PRICE){
|
||||
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANG_AVG_PRICE){
|
||||
price_map.fang_avg_price = r.FANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.LIANJIA_AVG_PRICE){
|
||||
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.CITYHOUSE_AVG_PRICE){
|
||||
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.MAITIAN_AVG_PRICE){
|
||||
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.KE_AVG_PRICE){
|
||||
price_map.ke_avg_price = r.KE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TAOFANG_AVG_PRICE){
|
||||
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ZHONGYUAN_AVG_PRICE){
|
||||
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TBS_AVG_PRICE){
|
||||
price_map.tbs_avg_price = r.TBS_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.QFANG_AVG_PRICE){
|
||||
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGTAN_AVG_PRICE){
|
||||
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGDUODUO_AVG_PRICE){
|
||||
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.WIWJ_AVG_PRICE){
|
||||
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
|
||||
}
|
||||
dataMap.updatetime = r.UPDATETIME
|
||||
|
||||
allRows << dataMap
|
||||
});
|
||||
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
|
||||
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def map = [:]
|
||||
for(def db : dbcursor){
|
||||
def key = db.region+"##"+db.name
|
||||
def dataMap = map.get(key)
|
||||
if(!dataMap){
|
||||
dataMap = [:]
|
||||
dataMap.put("id", db._id.toString())
|
||||
map.put(key, dataMap)
|
||||
}
|
||||
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
|
||||
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
|
||||
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
|
||||
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
|
||||
}
|
||||
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
|
||||
dataMap.put("soufang_esf_price", db.soufang_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
|
||||
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
|
||||
}
|
||||
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
|
||||
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
|
||||
dataMap.put("maitian_esf_price", db.maitian_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
|
||||
dataMap.put("beike_esf_price", db.beike_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
|
||||
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
|
||||
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
|
||||
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
|
||||
dataMap.put("qfang_esf_price", db.qfang_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
|
||||
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
|
||||
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
|
||||
}
|
||||
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
}
|
||||
|
||||
def resultMap = [:]
|
||||
map.each {key, value->
|
||||
if(value && value.size()>1){
|
||||
if(value.size()==2 && !value.lianjia_esf_price && !value.beike_esf_price){
|
||||
def price = statMap.get(key)
|
||||
if(price){
|
||||
def priceMap = [:]
|
||||
def startPrice = (int)(price - price*0.3)
|
||||
def endPrice = (int)(price*0.3 + price)
|
||||
value.each{pk,pv->
|
||||
if(pk.contains("price")){
|
||||
if(pv>=startPrice && pv<=endPrice){
|
||||
priceMap.put(pk,pv)
|
||||
}
|
||||
}else{
|
||||
priceMap.put(pk,pv)
|
||||
}
|
||||
}
|
||||
if(priceMap.size()>1){
|
||||
resultMap.put(key, priceMap)
|
||||
}
|
||||
}
|
||||
}else{
|
||||
resultMap.put(key, value)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return resultMap
|
||||
}
|
||||
|
||||
|
||||
def getDistrictStatByCity(city, weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"##"+d.district_name, d.avg_price)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.bigdata_third_party_price values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
pgsqlInstance.withBatch(50000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18])
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+31
-33
@@ -13,7 +13,7 @@ class MsbPocTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(小区、城市均价).xlsx"
|
||||
def inputfile = "D:\\数据\\建行\\房产估值\\_业务5月份系统反馈问题价格核查20240416-20240516 - 国信达0524.xlsx"
|
||||
// def inputfile = "D:\\数据\\美术宝\\美术宝测试数据(匹配V2).xlsx"//美术宝测试数据(匹配).xlsx
|
||||
// standard(inputfile)
|
||||
// match(inputfile)
|
||||
@@ -24,7 +24,7 @@ class MsbPocTest {
|
||||
// cityPrice()
|
||||
// mergeMatchPrice("D:\\数据\\美术宝\\美术宝-新接口价格.xlsx")
|
||||
|
||||
addCCBPrice("C:\\Users\\gaojuan\\Desktop\\合肥科技农商行测试.xlsx")
|
||||
addCCBPrice(inputfile)
|
||||
}
|
||||
|
||||
def cityPrice(){
|
||||
@@ -69,21 +69,21 @@ class MsbPocTest {
|
||||
|
||||
def matchOld(path){
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['S'] && row['D']){
|
||||
if(!row['Y'] && row['G']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def name = row['F']
|
||||
def address = row['H']
|
||||
def region = null //row['D']
|
||||
def name = row['E']
|
||||
def address = row['G']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
|
||||
def jsonObject = OpenApiClient.addressMatch(city, address, "property", false, region, name)
|
||||
if(jsonObject != null && jsonObject.precise!=-1) {
|
||||
row['S'] = jsonObject.get("id")
|
||||
row['T'] = jsonObject.get("city")
|
||||
row['U'] = jsonObject.get("district")
|
||||
row['V'] = jsonObject.get("name")
|
||||
row['W'] = jsonObject.get("propertyType")
|
||||
row['X'] = jsonObject.get("credit")
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
row['AC'] = jsonObject.get("propertyType")
|
||||
// row['X'] = jsonObject.get("credit")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -111,23 +111,18 @@ class MsbPocTest {
|
||||
|
||||
def match(file){
|
||||
ExcelUtils.multiThreadingModify(file, { row ->
|
||||
if(!row['J'] && row['F'] && row['P']=="没匹配" ){
|
||||
if(!row['Y']){
|
||||
def city = row['C']
|
||||
def region = row['D']
|
||||
def region = null //row['B']
|
||||
def address = row['G'] //row['G']//row['F']
|
||||
|
||||
println "[match] city:"+city+" region:"+region+" address:"+address
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, null, false)
|
||||
if(jsonObject != null && jsonObject.credit>0) {
|
||||
row['J'] = jsonObject.get("id")
|
||||
row['K'] = jsonObject.get("city")
|
||||
row['L'] = jsonObject.get("district")
|
||||
row['M'] = jsonObject.get("name")
|
||||
def type = jsonObject.get("propertyType")
|
||||
if(type && type!='null'){
|
||||
row['N'] = type
|
||||
}
|
||||
row['O'] = jsonObject.get("credit")
|
||||
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
|
||||
if(jsonObject != null) {
|
||||
row['Y'] = jsonObject.get("id")
|
||||
row['Z'] = jsonObject.get("city")
|
||||
row['AA'] = jsonObject.get("district")
|
||||
row['AB'] = jsonObject.get("name")
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -174,19 +169,22 @@ class MsbPocTest {
|
||||
def addCCBPrice(path){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(path, { row ->
|
||||
if(!row['M']){
|
||||
def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
if(dist){
|
||||
def d = CommonDatautils.findDistrictById(dist._id.toString())
|
||||
if(!row['V']){
|
||||
// def dist = CommonDatautils.findDistrict(row['E'], row['H'], row['I'], "district")
|
||||
// if(dist){
|
||||
if(row['Y'] instanceof String){
|
||||
def d = CommonDatautils.findDistrictById(row['Y'])
|
||||
if(d){
|
||||
def cityId = cityCodeDataMap.get(d.city)
|
||||
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
|
||||
def map = CcbCommonUtils.findAvgPrice(d._id.toString(), cityId)
|
||||
if(map){
|
||||
row['M'] = map.avg_price
|
||||
row['V'] = map.avg_price
|
||||
// row['I'] = map.calc_type
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+46
-29
@@ -33,17 +33,31 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "C:\\Users\\gaojuan\\Desktop\\7Ô³ɽ»°¸Àýbadcase.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\神经网络计算.xlsx"
|
||||
addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
exportPkPrice()
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
// addTradePrice(filePath)
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addTradePrice(filePath){
|
||||
ExcelUtils.multiThreadingModify(filePath, { d ->
|
||||
if(d['Y']){
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
def tradePrice = CommonDatautils.findTrades(dist, '2024-02-01', '2024-05-01')
|
||||
if (tradePrice) {
|
||||
d['W'] = tradePrice.toInteger()
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
def addRegion(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d->
|
||||
def dist = CommonDatautils.findDistrictById(d['A'])
|
||||
@@ -153,15 +167,15 @@ class AddPriceTest {
|
||||
}
|
||||
|
||||
def addCcbPrice(filepath){
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['M']){
|
||||
def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['M'], cityId)
|
||||
if(d['A']){
|
||||
// def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
|
||||
if(map){
|
||||
d['AK'] = map.avg_price
|
||||
d['AL'] = map.calc_type
|
||||
d['AM'] = map.credit_value
|
||||
// d['AK'] = map.avg_price
|
||||
d['P'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -190,11 +204,11 @@ class AddPriceTest {
|
||||
|
||||
def addThirdPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['M'])
|
||||
if (!d['X']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['Y'])
|
||||
if(dist){
|
||||
def thirdPrice = findThirdPartyPrice(dist.city, dist.region, dist.name)
|
||||
d['AN'] = thirdPrice
|
||||
d['X'] = thirdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -206,8 +220,8 @@ class AddPriceTest {
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", new BasicDBObject('$gte', 202212).append('$lte',202302))
|
||||
|
||||
// q.put("s_date", new BasicDBObject('$gte', 202311).append('$lte',202404))
|
||||
q.put("s_date", 202404)
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
|
||||
def price = null
|
||||
for(def db : dbcursor){
|
||||
@@ -216,10 +230,10 @@ class AddPriceTest {
|
||||
break;
|
||||
}
|
||||
|
||||
if(!price && db.beike_esf_price && db.beike_esf_price>2000){
|
||||
price = db.beike_esf_price
|
||||
break
|
||||
}
|
||||
// if(!price && db.beike_esf_price && db.beike_esf_price>2000){
|
||||
// price = db.beike_esf_price
|
||||
// break
|
||||
// }
|
||||
}
|
||||
return price
|
||||
}
|
||||
@@ -227,12 +241,12 @@ class AddPriceTest {
|
||||
|
||||
def addGXDPrice(filepath){
|
||||
ExcelUtils.multiThreadingModify(filepath, { d ->
|
||||
if (d['B']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if (!d['M']) {
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
def gxdPrice = findGXDPrice(dist.city, dist.region, dist.name)
|
||||
// def gxdPrice = findGXDPrice(d['D'], d['E'], d['C'])
|
||||
d['L'] = gxdPrice
|
||||
d['M'] = gxdPrice
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -256,14 +270,17 @@ class AddPriceTest {
|
||||
|
||||
def addFjwPrice(path){
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
if(!d['K']){
|
||||
def dist = CommonDatautils.findDistrictById(d['E'])
|
||||
if(dist){
|
||||
if(!d['L']){
|
||||
if(d['B'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['B'])
|
||||
if(dist){
|
||||
// d['N'] = dist.region
|
||||
def dbo = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2023-07-16","yyyy-MM-dd"))
|
||||
if(dbo){
|
||||
// d['K'] = dbo.avg_price
|
||||
d['K'] = dbo.avg_price_2b
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
// d['W'] = dbo.avg_price
|
||||
// d['X'] = dbo.cal_type
|
||||
d['L'] = dbo.avg_price_2b
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+63
-163
@@ -4,6 +4,7 @@ import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||
import groovy.json.JsonOutput
|
||||
import net.sf.json.JSONObject
|
||||
import okhttp3.*;
|
||||
import java.io.IOException;
|
||||
@@ -15,177 +16,76 @@ import java.util.concurrent.TimeUnit;
|
||||
import okhttp3.Request.Builder;
|
||||
|
||||
class QiliangTest {
|
||||
String proxyHost = 'tunnel2.qg.net';
|
||||
Integer proxyPort = 14186;
|
||||
String proxyUsername = '113BBC06';
|
||||
String proxyPassword = '3B2FBD05B0AF';
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
|
||||
// static main(args) {
|
||||
// def task = new QiliangTest()
|
||||
//// task.run()
|
||||
// task.getHttpResult("扬州扬子江北路399号11041")
|
||||
// }
|
||||
static main(args) {
|
||||
QiliangTest task = new QiliangTest();
|
||||
// task.run()
|
||||
|
||||
def run(){//地址测试-2024.3.4-结果.xlsx
|
||||
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
|
||||
if(!row['N']){
|
||||
def address = null
|
||||
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
|
||||
address = row['B']+row['C']
|
||||
}else{
|
||||
address = row['C']
|
||||
}
|
||||
if(address){
|
||||
def result = getHttpResult(address)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
if(data.province && data.province!='null'){
|
||||
row['N'] = data.province
|
||||
}
|
||||
if(data.city && data.city!='null'){
|
||||
row['O'] = data.city
|
||||
}
|
||||
if(data.district && data.district!='null'){
|
||||
row['P'] = data.district
|
||||
}
|
||||
if(data.town && data.town!='null'){
|
||||
row['Q'] = data.town
|
||||
}
|
||||
if(data.village && data.village!='null'){
|
||||
row['R'] = data.village
|
||||
}
|
||||
|
||||
if(data.areaCode && data.areaCode!='null'){
|
||||
row['S'] = data.areaCode
|
||||
}
|
||||
|
||||
if(data.farmer == "0"){
|
||||
row['T'] = "非农"
|
||||
}else{
|
||||
row['T'] = "农"
|
||||
}
|
||||
if(data.isGovt == "0"){
|
||||
row['U'] = "否"
|
||||
}else{
|
||||
row['U'] = "是"
|
||||
}
|
||||
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
row['V'] = s
|
||||
}
|
||||
if(data.govtAddr && data.govtAddr!='null'){
|
||||
row['W'] = data.govtAddr?data.govtAddr:''
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
def data = [
|
||||
'address': '福建省闽侯县上街镇乌龙江南大道30号福州清华紫光科技园-海峡科技研发区1-3#研发楼12层1201研发',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
print(s);
|
||||
}
|
||||
|
||||
def getHttpResult(address){
|
||||
def data = new JSONObject()
|
||||
data.put("address", address)
|
||||
data.put("type","2")
|
||||
|
||||
Map headers = new HashMap();
|
||||
headers.put("Accept", "application/json, text/plain, */*");
|
||||
headers.put("Accept-Encoding", "gzip, deflate");
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
headers.put("Content-Type","application/json;charset=UTF-8")
|
||||
headers.put("Host","146.56.217.44:9010")
|
||||
headers.put("Origin","http://146.56.217.44:8088")
|
||||
headers.put("Proxy-Connection","keep-alive")
|
||||
headers.put("Referer","http://146.56.217.44:8088/")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
||||
|
||||
HttpRequest request1 = new HttpRequest()
|
||||
try{
|
||||
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
println response
|
||||
return response
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
|
||||
String getProxyResult(data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
String jsonData = new JsonOutput().toJson(data);
|
||||
// 创建代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort));
|
||||
|
||||
// 创建 URL 对象
|
||||
URL url = new URL(targetUrl);
|
||||
|
||||
// 创建连接对象
|
||||
URLConnection connection = url.openConnection(proxy);
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST");
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true);
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json");
|
||||
|
||||
// 发送请求参数
|
||||
OutputStream outputStream = connection.getOutputStream();
|
||||
outputStream.write(jsonData.getBytes("UTF-8"));
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
|
||||
// 读取响应
|
||||
int responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(connection.getInputStream()));
|
||||
StringBuilder response = new StringBuilder();
|
||||
String line;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line);
|
||||
}
|
||||
reader.close();
|
||||
return response.toString();
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
final static String proxyIp = "tunnel2.qg.net";
|
||||
final static Integer proxyPort = 14186;
|
||||
final static String authKey = "113BBC06";
|
||||
final static String password = "3B2FBD05B0AF";
|
||||
public static final MediaType JSON = MediaType.parse("application/json;charset=utf-8");
|
||||
|
||||
// public String request(params) throws IOException {
|
||||
// Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyIp, proxyPort));
|
||||
// OkHttpClient client = new OkHttpClient.Builder()
|
||||
// .proxy(proxy)
|
||||
// .proxyAuthenticator((route, response){
|
||||
// String credential = Credentials.basic(authKey, password);
|
||||
// return response.request().newBuilder().header("Proxy-Authorization", credential).build();
|
||||
// }).
|
||||
// build();
|
||||
//// Request request = new Request.Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").get().build();
|
||||
//
|
||||
// RequestBody body = RequestBody.create(JSON, JSON.toJSONString(params));
|
||||
// Request request = new Builder().url("http://146.56.217.44:9010/uams/farmer/all/v3/sign").post(body).build();
|
||||
//
|
||||
// Response response = client.newCall(request).execute();
|
||||
// return response.body().string();
|
||||
//
|
||||
//// return client.newCall(request).execute();
|
||||
// }
|
||||
public static void main(String[] args) {
|
||||
def proxyHost = "tunnel2.qg.net"
|
||||
def proxyPort = 14186
|
||||
def proxyUsername = '113BBC06'
|
||||
def proxyPassword = '3B2FBD05B0AF'
|
||||
|
||||
def targetUrl = 'http://146.56.217.44:9010/uams/farmer/all/v3/sign'
|
||||
def queryParams = ['address': '扬州扬子江北路399号11041', 'type': '2']
|
||||
|
||||
|
||||
// 创建代理服务器的连接
|
||||
def proxy = new URL("http://${proxyUsername}:${proxyPassword}@${proxyHost}:${proxyPort}")
|
||||
|
||||
// 打开代理连接
|
||||
def proxyConnection = proxy.openConnection()
|
||||
|
||||
// 创建目标URL的连接
|
||||
def url = new URL(targetUrl)
|
||||
def connection = url.openConnection(proxy)
|
||||
|
||||
// 设置请求方法
|
||||
connection.setRequestMethod("GET")
|
||||
|
||||
// 添加请求头(可选)
|
||||
connection.setRequestProperty("Content-Type", "application/json")
|
||||
|
||||
// 构建查询参数
|
||||
def queryString = queryParams.collect { k, v -> "${k}=${v}" }.join('&')
|
||||
|
||||
// 将查询参数附加到目标URL
|
||||
def finalUrl = "${targetUrl}?${queryString}"
|
||||
|
||||
// 发送请求
|
||||
connection.connect()
|
||||
|
||||
// 获取响应
|
||||
def responseCode = connection.getResponseCode()
|
||||
def responseMessage = connection.getResponseMessage()
|
||||
|
||||
// 读取响应内容
|
||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
|
||||
def responseContent = reader.readLine()
|
||||
reader.close()
|
||||
|
||||
println "Response Code: ${responseCode}"
|
||||
println "Response Message: ${responseMessage}"
|
||||
println "Response Content: ${responseContent}"
|
||||
}
|
||||
}
|
||||
|
||||
+66
-68
@@ -18,77 +18,75 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun
|
||||
|
||||
//def s = getTownshipInfo("118.702988,39.740979")
|
||||
def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx"
|
||||
//ExcelUtils.multiThreadingModify(filePath, { row ->
|
||||
//// if(!row['I']){
|
||||
//// def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"ÈËÃñÕþ¸®")
|
||||
//// if(rets){
|
||||
//// def obj = findOneByName(rets, row['C']+"ÈËÃñÕþ¸®")
|
||||
//// if(obj){
|
||||
//// row['I'] = obj.lng
|
||||
//// row['J'] = obj.lat
|
||||
//// println row['C'] + " "+obj
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// if(row['D'] && !row['F']){
|
||||
//// def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||
//// def map = ["lon":row['D'], "lat":row['E']]
|
||||
//// def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
|
||||
//// row['F'] = township
|
||||
//// }
|
||||
//
|
||||
//// if(row['F'] && !row['J']){
|
||||
//// def q = new BasicDBObject()
|
||||
//// q.put("city_name", row['B'])
|
||||
//// q.put("area_name", row['C'])
|
||||
//// def streetName = row['F']
|
||||
//// q.put("street_name", streetName)
|
||||
//// def count = COMMUNITY_CONN.count(q);
|
||||
//// row['J'] = count
|
||||
////// row['H'] = streetName
|
||||
//// }
|
||||
//// if(!row['H']){
|
||||
//// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
|
||||
//// if(gaodeMap && gaodeMap.township){
|
||||
//// row['H'] = gaodeMap.township
|
||||
//// }
|
||||
//// }
|
||||
//
|
||||
//// if(row['J'] && row['J'].toString().toDouble()>0){
|
||||
//// def up_query = new BasicDBObject()
|
||||
//// up_query.put("city_name", row['B'])
|
||||
//// up_query.put("area_name", row['C'])
|
||||
//// up_query.put("street_name", row['F'])
|
||||
//// def udata = new BasicDBObject()
|
||||
//// udata.put("has_gov", "ÊÇ")
|
||||
//// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
|
||||
//// }
|
||||
//
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
for(def d : objs){
|
||||
if(!d.name_pinyin){
|
||||
if(d.name){
|
||||
println d._id
|
||||
def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("name_pinyin", py1)
|
||||
if(d.location){
|
||||
def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
|
||||
def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
|
||||
udata.put("location", ["lat":lat, "lon":lon])
|
||||
ExcelUtils.modify(filePath, { row ->
|
||||
if(!row['I'] && row['I']=='宝应县'){
|
||||
def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府")
|
||||
if(rets){
|
||||
def obj = findOneByName(rets, row['C']+"人民政府")
|
||||
if(obj){
|
||||
row['I'] = obj.lng
|
||||
row['J'] = obj.lat
|
||||
println row['C'] + " "+obj
|
||||
}
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", d._id)
|
||||
COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(row['D'] && !row['F']){
|
||||
def townMap = findTownRange(row['B'], TOWN_POLYLINECOLL)
|
||||
def map = ["lon":row['D'], "lat":row['E']]
|
||||
def township = InsidePolygon.isInArea2(map.lon, map.lat, townMap)
|
||||
row['F'] = township
|
||||
}
|
||||
|
||||
// if(row['F'] && !row['J']){
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("city_name", row['B'])
|
||||
// q.put("area_name", row['C'])
|
||||
// def streetName = row['F']
|
||||
// q.put("street_name", streetName)
|
||||
// def count = COMMUNITY_CONN.count(q);
|
||||
// row['J'] = count
|
||||
//// row['H'] = streetName
|
||||
// }
|
||||
// if(!row['H']){
|
||||
// def gaodeMap = getTownshipInfo(row['D']+","+row['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// row['H'] = gaodeMap.township
|
||||
// }
|
||||
// }
|
||||
|
||||
// if(row['J'] && row['J'].toString().toDouble()>0){
|
||||
// def up_query = new BasicDBObject()
|
||||
// up_query.put("city_name", row['B'])
|
||||
// up_query.put("area_name", row['C'])
|
||||
// up_query.put("street_name", row['F'])
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("has_gov", "是")
|
||||
// COMMUNITY_CONN.update(up_query, new BasicDBObject('$set', udata), false, true)
|
||||
// }
|
||||
})
|
||||
|
||||
|
||||
//def objs = COMMUNITY_CONN.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT);
|
||||
//for(def d : objs){
|
||||
// if(!d.name_pinyin){
|
||||
// if(d.name){
|
||||
// println d._id
|
||||
// def py1 = com.fangjia.core.util.DataUtil.hanyuQuanpin(d.name, " ")
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("name_pinyin", py1)
|
||||
// if(d.location){
|
||||
// def lon = DataHandleUtils.getDoubleRound(6, d.location.lon)
|
||||
// def lat = DataHandleUtils.getDoubleRound(6, d.location.lat)
|
||||
// udata.put("location", ["lat":lat, "lon":lon])
|
||||
// }
|
||||
//
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("_id", d._id)
|
||||
// COMMUNITY_CONN.update(q, new BasicDBObject('$set', udata), false, true)
|
||||
// }
|
||||
// }
|
||||
//}
|
||||
|
||||
|
||||
def getTownshipInfo(def glocation){
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
package com.fangjia.script.gj.village
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.temp.shenjian.util.HttpUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.HttpClient
|
||||
import groovy.json.JsonOutput
|
||||
import net.sf.json.JSONObject
|
||||
import okhttp3.*;
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.Proxy;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
|
||||
import okhttp3.Request.Builder;
|
||||
|
||||
class QiliangTest1 {
|
||||
def proxyHost = 'tunnel2.qg.net'
|
||||
def proxyPort = 14186
|
||||
def proxyUsername = '113BBC06'
|
||||
def proxyPassword = '3B2FBD05B0AF'
|
||||
String targetUrl = "http://146.56.217.44:9010/uams/farmer/all/v3/sign";
|
||||
|
||||
static main(args) {
|
||||
def task = new QiliangTest()
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '江苏省扬州市宝应县安宜镇宝应大道88号',
|
||||
'type': '1'
|
||||
]
|
||||
def s = task.getProxyResult(data)
|
||||
print(s)
|
||||
}
|
||||
|
||||
def run(){//地址测试-2024.3.4-结果.xlsx
|
||||
ExcelUtils.modify("D:\\数据\\省联社\\对比.xlsx", { row ->
|
||||
if(!row['N']){
|
||||
def address = null
|
||||
if(row['B'] && row['C'] && !row['C'].toString().contains(row['B'])){
|
||||
address = row['B']+row['C']
|
||||
}else{
|
||||
address = row['C']
|
||||
}
|
||||
if(address){
|
||||
def result = getHttpResult(address)
|
||||
if(result && result.code==20000){
|
||||
def data = result.data
|
||||
if(data){
|
||||
if(data.province && data.province!='null'){
|
||||
row['N'] = data.province
|
||||
}
|
||||
if(data.city && data.city!='null'){
|
||||
row['O'] = data.city
|
||||
}
|
||||
if(data.district && data.district!='null'){
|
||||
row['P'] = data.district
|
||||
}
|
||||
if(data.town && data.town!='null'){
|
||||
row['Q'] = data.town
|
||||
}
|
||||
if(data.village && data.village!='null'){
|
||||
row['R'] = data.village
|
||||
}
|
||||
|
||||
if(data.areaCode && data.areaCode!='null'){
|
||||
row['S'] = data.areaCode
|
||||
}
|
||||
|
||||
if(data.farmer == "0"){
|
||||
row['T'] = "非农"
|
||||
}else{
|
||||
row['T'] = "农"
|
||||
}
|
||||
if(data.isGovt == "0"){
|
||||
row['U'] = "否"
|
||||
}else{
|
||||
row['U'] = "是"
|
||||
}
|
||||
|
||||
def list = data.indusParkList
|
||||
if(list && list!="null"){
|
||||
def s = ""
|
||||
for(def r : list){
|
||||
s = s + r.indusParkName +"-"+r.indusParkLevel+"\n"
|
||||
}
|
||||
row['V'] = s
|
||||
}
|
||||
if(data.govtAddr && data.govtAddr!='null'){
|
||||
row['W'] = data.govtAddr?data.govtAddr:''
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
JSONObject getHttpResult(address){
|
||||
JSONObject data = new JSONObject()
|
||||
data.put("address", address)
|
||||
data.put("type","2")
|
||||
|
||||
Map headers = new HashMap();
|
||||
headers.put("Accept", "application/json, text/plain, */*");
|
||||
headers.put("Accept-Encoding", "gzip, deflate");
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
headers.put("Content-Type","application/json;charset=UTF-8")
|
||||
headers.put("Host","146.56.217.44:9010")
|
||||
headers.put("Origin","http://146.56.217.44:8088")
|
||||
headers.put("Proxy-Connection","keep-alive")
|
||||
headers.put("Referer","http://146.56.217.44:8088/")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36")
|
||||
|
||||
HttpRequest request1 = new HttpRequest()
|
||||
try{
|
||||
String res = request1.postWithBody("http://146.56.217.44:9010/uams/farmer/all/v3/sign", JSONObject.fromObject(data.toString()), headers, "UTF-8")
|
||||
JSONObject response = JSONObject.fromObject(res)
|
||||
println response
|
||||
return response
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
def getProxyResult(data){
|
||||
// 将 JSON 对象转换为字符串
|
||||
def jsonData = new JsonOutput().toJson(data)
|
||||
// 创建代理对象
|
||||
def proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyHost, proxyPort))
|
||||
|
||||
// 创建 URL 对象
|
||||
def url = new URL(targetUrl)
|
||||
|
||||
// 创建连接对象
|
||||
def connection = url.openConnection(proxy)
|
||||
|
||||
// 设置代理认证
|
||||
java.net.Authenticator.setDefault(new java.net.Authenticator() {
|
||||
protected PasswordAuthentication getPasswordAuthentication() {
|
||||
return new PasswordAuthentication(proxyUsername, proxyPassword.toCharArray())
|
||||
}
|
||||
})
|
||||
|
||||
// 设置请求方法为 POST
|
||||
connection.setRequestMethod("POST")
|
||||
|
||||
// 允许输出流
|
||||
connection.setDoOutput(true)
|
||||
|
||||
// 设置请求头
|
||||
connection.setRequestProperty("Content-Type", "application/json")
|
||||
|
||||
// 发送请求参数
|
||||
def outputStream = connection.getOutputStream()
|
||||
outputStream.write(jsonData.getBytes("UTF-8"))
|
||||
outputStream.flush()
|
||||
outputStream.close()
|
||||
|
||||
// 读取响应
|
||||
def responseCode = connection.getResponseCode()
|
||||
if (responseCode == HttpURLConnection.HTTP_OK) {
|
||||
def reader = new BufferedReader(new InputStreamReader(connection.getInputStream()))
|
||||
def response = new StringBuilder()
|
||||
String line
|
||||
while ((line = reader.readLine()) != null) {
|
||||
response.append(line)
|
||||
}
|
||||
reader.close()
|
||||
return response.toString()
|
||||
} else {
|
||||
println "HTTP request failed with response code ${responseCode}"
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
}
|
||||
+218
-4
@@ -2,6 +2,7 @@ package com.fangjia.script.gj.xiamen
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
@@ -18,6 +19,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
@@ -50,7 +52,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_new", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
|
||||
DBCollection MAPPOI_CONN_TEST = MongoUtils.getCollectionByName2("fangjia", "map_poi_all_8", MAP_POI_URL)
|
||||
|
||||
@@ -58,6 +60,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
|
||||
def fullCityMap = null
|
||||
|
||||
DBCollection CHUN_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020)
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MAP_POI_URL)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new GaoDePoiDataProcess(), args)
|
||||
@@ -69,12 +74,215 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
// synchMapPoi();
|
||||
|
||||
|
||||
synchCompany()
|
||||
// synchCompany()
|
||||
// poiCatProcess()
|
||||
// addressAndCatProcess()
|
||||
// synchNetwork()
|
||||
// println DateUtils.parseFromTightByPattern("2023-11-30","yyyy-MM-dd").getTime()
|
||||
// sysnchPoi()
|
||||
// sysnchVillage()
|
||||
|
||||
addTownship()
|
||||
}
|
||||
|
||||
def addTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\小区\\河北省小区.xlsx",{d->
|
||||
if(d['E']){
|
||||
// def gaodeMap = getTownshipInfo(d['F']+","+d['E'])
|
||||
// if(gaodeMap && gaodeMap.township){
|
||||
// if(gaodeMap.city && gaodeMap.city!="[]"){
|
||||
// d['H'] = gaodeMap.city
|
||||
// }
|
||||
// if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
// d['I'] = gaodeMap.district
|
||||
// }
|
||||
// if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
// d['J'] = gaodeMap.township
|
||||
// }
|
||||
// if(gaodeMap.format_addr && gaodeMap.format_addr !='[]'){
|
||||
// d['K'] = gaodeMap.format_addr
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['H'])
|
||||
q.put("district", d['I'])
|
||||
q.put("street",d['J'])
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL_100.findOne(q)
|
||||
if(obj){
|
||||
d['L'] = "是"
|
||||
}
|
||||
}
|
||||
},0)
|
||||
|
||||
}
|
||||
|
||||
def sysnchVillage(){
|
||||
def buducitys = ["乌兰察布市","巴彦淖尔市","渭南市","宝鸡市","枣庄市","日照市","东营市","威海市","太原市","文山壮族苗族自治州","中卫市","临夏回族自治州","辽源市","抚顺市","阿坝藏族羌族自治州","宜宾市","中山市","亳州市","滁州市","宣城市","廊坊市","宁德市","龙岩市","厦门市","莆田市","天水市","清远市","湛江市","阳江市","河源市","潮州市","来宾市","百色市","防城港市","铜仁地区","毕节地区","承德市","衡水市","濮阳市","开封市","焦作市","三门峡市","平顶山市","信阳市","鹤壁市","十堰市","荆门市","武汉市","常德市","岳阳市","娄底市","株洲市","盐城市","苏州市","景德镇市","抚州市","本溪市","盘锦市","包头市","阿拉善盟","榆林市","铜川市","西安市","临沂市","滨州市","青岛市","朔州市","晋中市","巴中市","绵阳市","广安市","资阳市","固原市","甘南藏族自治州","内江市","曲靖市","淮南市","巢湖市","黄山市","淮北市","三明市","漳州市","陇南市","广州市","云浮市","揭阳市","贺州市","南宁市","遵义市","安顺市","张家口市","唐山市","邢台市","安阳市","郑州市","驻马店市","宜昌市","黄冈市","益阳市","邵阳市","湘西土家族苗族自治州","郴州市","泰州市","宿迁市","宜春市","鹰潭市","朝阳市","营口市","丹东市","鄂尔多斯市","延安市","商洛市","济宁市","潍坊市","济南市","上海市","晋城市","南充市","北海市","赤峰市","六安市","池州市","福州市","惠州市","江门市","汕头市","梧州市","柳州市","黔南布依族苗族自治州","保定市","周口市","南阳市","孝感市","黄石市","张家界市","湘潭市","永州市","南京市","徐州市","无锡市","吉安市","葫芦岛市","鞍山市","呼和浩特市","吴忠市","咸阳市","安康市","泰安市","烟台市","吕梁市","运城市","广元市","遂宁市","泸州市","天津市","石嘴山市","昭通市","铜陵市","肇庆市","汕尾市","深圳市","贵港市","黔东南苗族侗族自治州","黔西南布依族苗族自治州","漯河市","扬州市","连云港市","常州市","九江市","萍乡市","辽阳市","汉中市","菏泽市","淄博市","大同市","长治市","阳泉市","马鞍山市","平凉市","银川市","玉林市","咸宁市","怀化市","上饶市","赣州市","聊城市","忻州市","临汾市","达州市","宿州市","随州市","德州市","恩施土家族苗族自治州","阿拉尔市","石河子市","五家渠市","图木舒克市","定安县","儋州市","万宁市","保亭黎族苗族自治县","西沙群岛","济源市","潜江市","中沙群岛","南沙群岛","屯昌县","昌江黎族自治县","陵水黎族自治县","五指山市","仙桃市","琼中黎族苗族自治县","乐东黎族自治县","临高县","琼海市","白沙黎族自治县","东方市","天门市","神农架林区","澄迈县","文昌市","澳门特别行政区","香港特别行政区","桃园市","台北市","南投县","嘉义市","彰化县","新竹县","澎湖县","台东县","宜兰县","新北市","基隆市","屏东县","嘉义县","云林县","花莲县","台南市","台中市","新竹市","高雄市","苗栗县"]
|
||||
for(def city : buducitys){
|
||||
def pois = findAllVillagePoiMap(city)
|
||||
def rows = createVillageData(city, pois)
|
||||
|
||||
save(city, rows)
|
||||
saveToMG(city, rows)
|
||||
println city +" "+pois.size()+" add:"+rows.size()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def createVillageData(city, pois){
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(pois.size())
|
||||
def townMap = findTownRange(city)
|
||||
|
||||
for(def i=0;i<pois.size();i++){
|
||||
def p = pois[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def obj = MAPPOI_CONN.findOne(new BasicDBObject("name", p.name).append("district", p.district).append("city", p.city))
|
||||
if(!obj){
|
||||
def prov = ProvinceData.getProvince(p.city.replace("市",""))
|
||||
|
||||
def ret = createVillageData(prov, p, townMap)
|
||||
rets.addAll(ret)
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
def isTown( city, region, name){
|
||||
// ElasticUtils.executeSqlQuery("select * from administrative_town where province = '${prov}' and city='${city}' and district='${region}' and town='${town}'", {
|
||||
// obj = it
|
||||
// },"192.168.0.89")
|
||||
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_town/_search"
|
||||
def body = """
|
||||
{
|
||||
"query": {
|
||||
"bool":{
|
||||
"must":[
|
||||
{"term": {"city.keyword":"${city}"}},
|
||||
{"term": {"district.keyword":"${region}"}},
|
||||
{"term": {"town.keyword":"${town}"}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
def response = JSONObject.fromObject(res)
|
||||
def obj = null
|
||||
if(response && response.hits){
|
||||
def hits = response.hits.hits
|
||||
if(hits){
|
||||
obj = hits[0]
|
||||
}
|
||||
}
|
||||
println(response)
|
||||
return obj
|
||||
}
|
||||
|
||||
|
||||
def createVillageData(province, map, townMap){
|
||||
// def fjw_province = province
|
||||
// def fjw_city = RegionData.formatCity(map.city)
|
||||
// def fjw_region = RegionData.formatRegion(fjw_city, map.district)
|
||||
|
||||
def retMap = [:]
|
||||
retMap.province = ProvinceData.getFullProvince(province)
|
||||
// retMap.fjw_province = fjw_province
|
||||
// retMap.fjw_city = fjw_city
|
||||
|
||||
retMap.city = map.city
|
||||
retMap.district = map.district
|
||||
retMap.name = map.name
|
||||
retMap.type = "地名地址信息"
|
||||
retMap.secondary_type = "普通地名"
|
||||
retMap.small_type = "村庄级地名"
|
||||
retMap.cat = "地名地址信息;普通地名;村庄级地名"
|
||||
|
||||
|
||||
retMap.location = ["lon": map.gcj02_x,"lat":map.gcj02_y]
|
||||
retMap.township = InsidePolygon.isInArea2(retMap.location.lon, retMap.location.lat, townMap)
|
||||
//高德转百度
|
||||
def baiduMap = GeoUtils.gcj02ToBd09(retMap.location.lat, retMap.location.lon)
|
||||
if(baiduMap){
|
||||
def location = [:]
|
||||
location.lon = DataHandleUtils.getDoubleRound(6, baiduMap.lng)
|
||||
location.lat = DataHandleUtils.getDoubleRound(6, baiduMap.lat)
|
||||
retMap.bd_location = location
|
||||
}
|
||||
|
||||
//高德转WGS84
|
||||
def gpsLoactionMap = GeoUtils.gcjToGps84(retMap.location.lat, retMap.location.lon)
|
||||
if(gpsLoactionMap){
|
||||
def gpslocation = [:]
|
||||
gpslocation.lon = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lng)
|
||||
gpslocation.lat = DataHandleUtils.getDoubleRound(6, gpsLoactionMap.lat)
|
||||
retMap.gps_location = gpslocation
|
||||
}
|
||||
|
||||
if(!retMap.township){
|
||||
|
||||
def gaodeMap = getTownshipInfo(retMap.location.lon+","+retMap.location.lat)
|
||||
if(gaodeMap && gaodeMap.township){
|
||||
retMap.township = gaodeMap.township
|
||||
}
|
||||
|
||||
// if(!fjw_region){
|
||||
// if(gaodeMap && gaodeMap.district){
|
||||
// fjw_region = RegionData.formatRegion(fjw_city, gaodeMap.district)
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(!fjw_region){
|
||||
// println fjw_city+" "+retMap.region+" 无法格式化 "+retMap.location + " "+gaodeMap
|
||||
// }
|
||||
|
||||
// retMap.fjw_district = fjw_region
|
||||
}
|
||||
// retMap.fjw_district = fjw_region
|
||||
retMap.spell_name = com.fangjia.core.util.DataUtil.hanyuQuanpin(retMap.name, " ")
|
||||
retMap.version = version
|
||||
retMap.data_source = "百度地图"
|
||||
retMap.data_id = map._id.toString()
|
||||
retMap.id = map._id.toString()
|
||||
|
||||
if(retMap.secondary_type && TAGS.contains(retMap.secondary_type)){
|
||||
retMap.status = 0
|
||||
}else{
|
||||
retMap.status = 1
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
|
||||
def findAllVillagePoiMap(city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
keys.put("addr", 1)
|
||||
keys.put("gcj02_x", 1)
|
||||
keys.put("gcj02_y", 1)
|
||||
def list = CHUN_POI_COLL.find(new BasicDBObject("std_tag","行政地标;村庄").append("city", city), keys).toList()
|
||||
return list;
|
||||
}
|
||||
|
||||
def sysnchPoi(){
|
||||
@@ -158,7 +366,6 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
final CountDownLatch latch = new CountDownLatch(results.size())
|
||||
|
||||
def prov = ProvinceData.getProvince(city.replace("ÊÐ",""))
|
||||
|
||||
for(def i=0;i<results.size();i++){
|
||||
def map = results[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
@@ -348,7 +555,7 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def client = ElasticUtils.getClient("192.168.0.89")
|
||||
for (int i = 0; i < rows.size(); i++) {
|
||||
def map = rows[i]
|
||||
request.add(new IndexRequest("map_poi", "_doc", map.id).source(map, XContentType.JSON));
|
||||
request.add(new IndexRequest("didata_map_poi_single", "_doc", map.id).source(map, XContentType.JSON));
|
||||
if(i%1000==0){
|
||||
print "."
|
||||
}
|
||||
@@ -409,6 +616,8 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
def getTownshipInfo(def glocation){
|
||||
def township = null
|
||||
def district = null
|
||||
def city = null
|
||||
def format_addr = null
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
@@ -420,6 +629,9 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
if(addressComponent){
|
||||
township = addressComponent.township
|
||||
district = addressComponent.district
|
||||
city = addressComponent.city
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -433,8 +645,10 @@ class GaoDePoiDataProcess extends BaseBatch{
|
||||
}
|
||||
|
||||
def map = [:]
|
||||
map.city = city
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
+17
-16
@@ -3,6 +3,7 @@ package com.fangjia.script.gj.xiamen
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.xingyebank.Data
|
||||
@@ -34,16 +35,18 @@ class PoiWash {
|
||||
public static def regions = []
|
||||
public static def nameKeys = null
|
||||
|
||||
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiWash()
|
||||
// task.updateId()
|
||||
// task.run()
|
||||
// task.synch()
|
||||
// task.check()
|
||||
task.deduplicate()
|
||||
// task.deduplicate()
|
||||
// task.updateEsMapPoi()
|
||||
// task.checkSpecialSymbols()
|
||||
// task.splitRoad()
|
||||
task.splitRoad()
|
||||
}
|
||||
|
||||
def updateId(){
|
||||
@@ -452,9 +455,10 @@ class PoiWash {
|
||||
|
||||
def splitRoad(){
|
||||
//"ÈýÃ÷ÊÐ",
|
||||
def citys = [ "漳州市","宁德市","泉州市","厦门市","福州市","南平市","龙岩市","莆田市","上海市","深圳市","北京市","广州市","杭州市","苏州市","成都市","重庆市"]
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
for(def city : citys) {
|
||||
def datas = findPoiByQuery(city, ['city','address'])
|
||||
def cityObj = fullCityMap.get(city)
|
||||
def datas = findPoiByQuery(cityObj.city, ['city','address'])
|
||||
splitRoadRun(datas)
|
||||
println city +" Íê³É"
|
||||
}
|
||||
@@ -472,18 +476,15 @@ class PoiWash {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.structure(map.city, map.address)
|
||||
if(jsonObject && jsonObject.containsKey("result")){
|
||||
def dataMap = jsonObject.get("result")
|
||||
if(dataMap.get("street")){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("street", dataMap.get("street"))
|
||||
if(dataMap.get("streetNumber")){
|
||||
udata.put("street_number", dataMap.get("streetNumber"))
|
||||
}
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.format(map.city, map.address)
|
||||
def street = jsonObject.get("street")
|
||||
def streetNumber = jsonObject.get("streetNumber")
|
||||
if(jsonObject && street && streetNumber && street!="null" && streetNumber!='null'){
|
||||
updateTotalCount.addAndGet(1)
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("address", jsonObject.get("street")+jsonObject.get("streetNumber"))
|
||||
udata.put("m_date", new Date())
|
||||
MAPPOI_CONN.update(new BasicDBObject('_id',map._id),new BasicDBObject('$set', udata))
|
||||
}
|
||||
println updateTotalCount.get()+" / "+totalCount.get()+" "+count +" "
|
||||
} catch (Exception e) {
|
||||
|
||||
+45
-21
@@ -5,6 +5,8 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
@@ -26,8 +28,8 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL)
|
||||
DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
|
||||
|
||||
DBCollection stat_administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "stat_administrative_division", MONGO100_URL)
|
||||
public static final DBCollection stat_administrative_division_206_coll = MongoUtils.getCollectionByName("crawl", "stats_administrative_division", "192.168.0.206", 27020)
|
||||
@@ -40,7 +42,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
static DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
|
||||
public static final DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2023", "192.168.0.206", 27020)
|
||||
|
||||
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
@@ -61,11 +63,13 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
run()
|
||||
// println "全国district, shop, office同步完成"
|
||||
//民政部编码上传
|
||||
//codeRun()
|
||||
// codeRun()
|
||||
|
||||
//println "民政部省,市,区同步完成"
|
||||
//println new ObjectId("622c433afb459ab08fdab460").getDate()
|
||||
|
||||
// addTest()
|
||||
// updateTownship()
|
||||
}
|
||||
|
||||
def addTest(){
|
||||
@@ -346,24 +350,44 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
|
||||
def codeRun(){
|
||||
|
||||
// def q = new BasicDBObject()
|
||||
// def dbcursor = mca_administrative_division_coll.find(q)
|
||||
// def rets = []
|
||||
// for(def it : dbcursor){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("_id", it._id)
|
||||
// udata.put("province", it.province)
|
||||
// udata.put("city", it.city)
|
||||
// udata.put("district", it.district)
|
||||
// udata.put("mca_province_code", it.province_code)
|
||||
// udata.put("mca_city_code", it.city_code)
|
||||
// udata.put("mca_district_code", it.district_code)
|
||||
// udata.put("level", it.level)
|
||||
// udata.put("c_date", it.c_date)
|
||||
// rets << udata
|
||||
//// administrative_division_100_CONN.save(udata)
|
||||
//// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||
// }
|
||||
// administrative_division_100_CONN.insert(rets)
|
||||
//
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = mca_administrative_division_coll.find(q)
|
||||
def rets = []
|
||||
for(def it : dbcursor){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("_id", it._id)
|
||||
udata.put("province", it.province)
|
||||
udata.put("city", it.city)
|
||||
udata.put("district", it.district)
|
||||
udata.put("mca_province_code", it.province_code)
|
||||
udata.put("mca_city_code", it.city_code)
|
||||
udata.put("mca_district_code", it.district_code)
|
||||
udata.put("level", it.level)
|
||||
udata.put("c_date", it.c_date)
|
||||
rets << udata
|
||||
// administrative_division_100_CONN.save(udata)
|
||||
// administrative_division_100_CONN.update(new BasicDBObject('_id', it._id), new BasicDBObject('$set', udata), true, false)
|
||||
q.put("province","澳门特别行政区")
|
||||
def dbcursor = administrative_division_2022_CONN.find(q)
|
||||
for(def d : dbcursor){
|
||||
administrative_division_100_CONN.save(d)
|
||||
|
||||
}
|
||||
administrative_division_100_CONN.insert(rets)
|
||||
}
|
||||
|
||||
def updateTownship(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\浙江省小区.xlsx", { row ->
|
||||
if(row['Q'] && row['Q']=='是'){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("township", row['J'])
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata))
|
||||
}
|
||||
})
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+18
-1
@@ -98,6 +98,23 @@ class InsidePolygon {
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
public static List<List<Point>> createPoint4(def pointList){
|
||||
List<List<Point>> alllist = new ArrayList<>();
|
||||
for(def pointArr : pointList){
|
||||
List<Point> list = new ArrayList<>();
|
||||
for(def arr : pointArr){
|
||||
Point point = new Point();
|
||||
double x = arr[0];
|
||||
double y = arr[1];
|
||||
point.setLon(x);
|
||||
point.setLat(y);
|
||||
list.add(point);
|
||||
}
|
||||
alllist << list
|
||||
}
|
||||
return alllist
|
||||
}
|
||||
|
||||
/**
|
||||
* 判定一个经纬度点与区域的关系
|
||||
@@ -284,7 +301,7 @@ class InsidePolygon {
|
||||
if(flag){
|
||||
def arrs = key.split("##")
|
||||
name = arrs[1];
|
||||
break;
|
||||
// break;
|
||||
}
|
||||
}
|
||||
if(flag){
|
||||
|
||||
+1
-1
@@ -46,7 +46,7 @@ import java.util.regex.Pattern
|
||||
* 8:華硊摯華硊梗靡脤梑
|
||||
* 9:華硊摯華硊梗靡(祥樓Е郖癹秶)脤梑
|
||||
* 10:⑽秞⑤饜
|
||||
*11:open½Ó¿ÚÆ¥Åä
|
||||
* 11:open½Ó¿ÚÆ¥Åä
|
||||
* 14:⑽秞⑤饜ㄛ祥樓華硊癹秶
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
|
||||
@@ -29,11 +29,11 @@ import static com.mongodb.client.model.Sorts.*
|
||||
public class DbUtils{
|
||||
|
||||
public static final String ONLINE_MONGODB_DOMAIN = "mongo.db.fangjia.com";
|
||||
public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
|
||||
//public static DBCollection OPERATION_LOG = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('operation_log');
|
||||
|
||||
public static final String COMMON_DB = "fangjia";
|
||||
public static final int COMMON_PORT = 27017;
|
||||
public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
|
||||
//public static DSCollection seawater = new DataStore("seawater.db.fangjia.com", 27777).getCollection("fangjia.seawater")
|
||||
public static DSCollection seaweed = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static DSCollection SEAWEED_SLAVE = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
public static DSCollection seabird = new DataStore("seabird.db.fangjia.com").getCollection("fangjia.seabird")
|
||||
|
||||
+24
-1
@@ -76,7 +76,6 @@ class ElasticUtils {
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@@ -101,6 +100,30 @@ class ElasticUtils {
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca"
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
|
||||
|
||||
/**
|
||||
* 冪帠僅蛌華硊<br>
|
||||
|
||||
+3
-2
@@ -6,6 +6,7 @@ import com.fangjia.dal.SdQuery;
|
||||
import com.fangjia.dal.SortParam;
|
||||
import com.fangjia.dal.common.DistrictKey;
|
||||
import com.fangjia.dal.mongo.model.AgencyDistrictTradeRank;
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
import com.fangjia.dal.mongo.model.DistrictFacilitys;
|
||||
import com.fangjia.dal.seaweed.model.District;
|
||||
import com.fangjia.service.bo.DistrictInfo;
|
||||
@@ -540,10 +541,10 @@ public interface DistrictService extends ModeService<DistrictKey, DistrictBo> {
|
||||
*
|
||||
* @param city
|
||||
* @param region
|
||||
* @param masterDistrictNname
|
||||
* @param masterDistrictName
|
||||
* @return
|
||||
*/
|
||||
List<District> findSlaves(String city, String region, String masterDistrictNname);
|
||||
List<District> findSlaves(String city, String region, String masterDistrictName);
|
||||
|
||||
/**
|
||||
* 分期操作时设置此小区为人工操作标识
|
||||
|
||||
+2
-3
@@ -2717,12 +2717,12 @@ public class DistrictServiceImpl implements DistrictService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<District> findSlaves(String city, String region, String masterDistrictNname) {
|
||||
public List<District> findSlaves(String city, String region, String masterDistrictName) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
params.add(new QueryParam("visible", 0));
|
||||
params.add(new QueryParam("city", city));
|
||||
params.add(new QueryParam("region", region));
|
||||
params.add(new QueryParam("masterDistrictNname", masterDistrictNname));
|
||||
params.add(new QueryParam("masterDistrictName", masterDistrictName));
|
||||
return districtDao.findAllByParams(params);
|
||||
}
|
||||
|
||||
@@ -2930,5 +2930,4 @@ public class DistrictServiceImpl implements DistrictService {
|
||||
params.add(new QueryParam("$sr", new SrQuery(query)));
|
||||
return districtDao.findAllByParams(params);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+4
-2
@@ -33,7 +33,7 @@ public interface DevelopmentZoneService {
|
||||
* @param id
|
||||
* @param udata
|
||||
*/
|
||||
public void updateDevelopmentZone(DevelopmentZone developmentZone);
|
||||
public void updateDevelopmentZone(Map<String,Object> udata, String id);
|
||||
|
||||
/**
|
||||
* 根据ID查询
|
||||
@@ -51,7 +51,9 @@ public interface DevelopmentZoneService {
|
||||
* @param pager
|
||||
* @return
|
||||
*/
|
||||
public List<DevelopmentZone> findDevelopmentZone(String province,String name, Pager pager);
|
||||
public List<DevelopmentZone> findDevelopmentZone(String province,String name,String level, Pager pager);
|
||||
|
||||
public long countDev(String province,String name,String level);
|
||||
|
||||
|
||||
}
|
||||
|
||||
+31
-5
@@ -10,6 +10,7 @@ import org.sothis.core.util.Pager;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
public class DevelopmentZoneServiceImpl implements DevelopmentZoneService {
|
||||
private DevelopmentZoneDao developmentZoneDao;
|
||||
@@ -26,8 +27,8 @@ public class DevelopmentZoneServiceImpl implements DevelopmentZoneService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateDevelopmentZone(DevelopmentZone developmentZone) {
|
||||
developmentZoneDao.update(developmentZone);
|
||||
public void updateDevelopmentZone(Map<String,Object> map,String id) {
|
||||
developmentZoneDao.updateById(map,id);
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -37,19 +38,44 @@ public class DevelopmentZoneServiceImpl implements DevelopmentZoneService {
|
||||
|
||||
|
||||
@Override
|
||||
public List<DevelopmentZone> findDevelopmentZone(String province, String name, Pager pager) {
|
||||
public List<DevelopmentZone> findDevelopmentZone(String province,String name,String level, Pager pager) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
if (StringUtils.isNotBlank(province)) {
|
||||
params.add(new QueryParam("province", province));
|
||||
Pattern pattern = Pattern.compile(".*" + province + ".*");
|
||||
params.add(new QueryParam("province", pattern));
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(name)) {
|
||||
params.add(new QueryParam("name", name));
|
||||
Pattern pattern = Pattern.compile(".*" + name + ".*");
|
||||
params.add(new QueryParam("name", pattern));
|
||||
}
|
||||
if (StringUtils.isNotBlank(level)) {
|
||||
Pattern pattern = Pattern.compile(".*" + level + ".*");
|
||||
params.add(new QueryParam("level", pattern));
|
||||
}
|
||||
|
||||
return developmentZoneDao.findAllByParams(params, pager.getStartRow(), pager.getPageSize());
|
||||
}
|
||||
|
||||
@Override
|
||||
public long countDev(String province, String name,String level) {
|
||||
List<QueryParam> params = new ArrayList<QueryParam>();
|
||||
if (StringUtils.isNotBlank(province)) {
|
||||
Pattern pattern = Pattern.compile(".*" + province + ".*");
|
||||
params.add(new QueryParam("province", pattern));
|
||||
}
|
||||
|
||||
if (StringUtils.isNotBlank(name)) {
|
||||
Pattern pattern = Pattern.compile(".*" + name + ".*");
|
||||
params.add(new QueryParam("name", pattern));
|
||||
}
|
||||
if (StringUtils.isNotBlank(level)) {
|
||||
Pattern pattern = Pattern.compile(".*" + level + ".*");
|
||||
params.add(new QueryParam("level", pattern));
|
||||
}
|
||||
return developmentZoneDao.findCountByParams(params);
|
||||
}
|
||||
|
||||
|
||||
public void setDevelopmentZoneDao(DevelopmentZoneDao developmentZoneDao) {
|
||||
this.developmentZoneDao = developmentZoneDao;
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
package com.fangjia.service.open;
|
||||
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2024/6/11
|
||||
*/
|
||||
public interface AliasFangService {
|
||||
|
||||
/**
|
||||
* 查询搜房别名与名称关系
|
||||
*
|
||||
* @param city
|
||||
* @param region
|
||||
* @param names
|
||||
* @return
|
||||
*/
|
||||
List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names);
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
package com.fangjia.service.open;
|
||||
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.mongo.dao.AliasFangDao;
|
||||
import com.fangjia.dal.mongo.model.AliasFang;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2024/6/11
|
||||
*/
|
||||
public class AliasFangServiceImpl implements AliasFangService{
|
||||
|
||||
private AliasFangDao aliasFangDao;
|
||||
|
||||
@Override
|
||||
public List<AliasFang> findFangByNameAndAlias(String city, String region, Set<String> names) {
|
||||
return aliasFangDao.findFangByNameAndAlias(city, region, names);
|
||||
}
|
||||
|
||||
public void setAliasFangDao(AliasFangDao aliasFangDao) {
|
||||
this.aliasFangDao = aliasFangDao;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user