This commit is contained in:
2024-08-22 17:00:20 +08:00
66 changed files with 9551 additions and 6186 deletions
@@ -1,11 +1,13 @@
package com.fangjia.bkoff.controllers;
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.util.DateUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.fangjia.bkoff.model.ApiUserModel;
import com.fangjia.bkoff.model.JsonModel;
import com.fangjia.core.util.BeanUtils;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.DateUtil;
import com.fangjia.core.util.StringUtil;
import com.fangjia.core.util.*;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.ApiInvokeStatistical;
import com.fangjia.dal.mongo.model.ApiUserCfg;
@@ -27,23 +29,33 @@ import com.fangjia.service.sql.UserService;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.time.DateUtils;
import org.apache.poi.hssf.usermodel.*;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.bson.types.ObjectId;
import org.json.JSONArray;
import org.sothis.core.util.CollectionUtils;
import org.sothis.web.mvc.ActionContext;
import org.sothis.web.mvc.annotation.Param;
import scala.Int;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.math.BigDecimal;
import java.net.URLEncoder;
import java.text.DecimalFormat;
import java.text.SimpleDateFormat;
import java.util.*;
/**
* api用户开通等管理controller
*
@@ -61,8 +73,6 @@ public class ApiUserController {
private ApiInvokeStatisticalService apiInvokeStatisticalService;
private ApiVisitCountService apiVisitCountService;
private ApiCountCityService apiCountCityService;
/**
* 所有api用户的信息展示
*
@@ -303,14 +313,15 @@ public class ApiUserController {
user.put("password", DigestUtils.md5Hex(model.getPassword()));
//
// // 更改用户密码
this.userService.updateById(user,userId);
this.userService.updateById(user, userId);
// 刷新缓存
this.apiUserService.clearApiUserAuthorityCache(userId);
model.setViewType("redirect");
model.setViewParam("location", "/apiUser/edit?userId="+userId);
model.setViewParam("location", "/apiUser/edit?userId=" + userId);
return model;
}
public Object editPwdAction(ApiUserModel model) {
Long userId = model.getUserId();
User user = this.userService.findById(userId);
@@ -975,7 +986,21 @@ public class ApiUserController {
} else {
cities = new ArrayList<String>();
}
List<OpenUriAnaly> analys = openUriAnalyService.findOpenUriAnalyLogs(model.getUserId(), null, startDate, endDate);
// 只查询正式账号
List<QueryParam> queryParams = Arrays.asList(new QueryParam("isTest", 0));
List<ApiUser> apiUsers = this.apiUserService.findApiUsersByParamsSortDate(queryParams);
// 获取正式账号的ids
List<Integer> ids = new ArrayList<Integer>();
for (ApiUser apiUser : apiUsers) {
ids.add((int) apiUser.getId());
}
List<OpenUriAnaly> analyAll = openUriAnalyService.findOpenUriAnalyLogs(model.getUserId(), null, startDate, endDate);
List<OpenUriAnaly> analys = new ArrayList<OpenUriAnaly>();
for (OpenUriAnaly analy : analyAll) {
if (ids.contains(analy.getApiUserId())) {
analys.add(analy);
}
}
List<StatAnalyBo> bos = null;
List<EachCityBo> eachCityBos = null;
if (model.getWay() == null) {
@@ -1016,9 +1041,7 @@ public class ApiUserController {
}
}
}
List<Long> userIds = new ArrayList<Long>();
for (StatAnalyBo boTemp : bos) {
userIds.add(Long.valueOf(boTemp.getUserId()));
List<ApiVisitCount> apiVisitCounts = apiVisitCountService.findAllApiVisitCount(Long.valueOf(boTemp.getUserId()), null, startDate, endDate);
int total = 0;
for (ApiVisitCount apiVisitCount : apiVisitCounts) {
@@ -1027,7 +1050,6 @@ public class ApiUserController {
boTemp.setBillingCount(total);
}
List<ApiUser> apiUsers = apiUserService.findApiUserByIds(userIds);
Map<Long, ApiUser> idUserMap = new HashMap<Long, ApiUser>();
for (ApiUser apiUser : apiUsers) {
idUserMap.put(apiUser.getId(), apiUser);
@@ -1051,8 +1073,199 @@ public class ApiUserController {
return model;
}
public Object exportAction(ApiUserModel model, HttpServletResponse response) throws IllegalAccessException {
Date startDate = null;
Date endDate = null;
if (StringUtils.isBlank(model.getBegDate()) && StringUtils.isBlank(model.getEndDate())) {
startDate = DateUtil.add(DateUtil.formatToYYYYMMDD(new Date()), -7);
endDate = DateUtil.parseStringToDate(DateUtil.formatToYYYYMMDD(new Date()));
} else {
if (StringUtils.isNotBlank(model.getBegDate())) {
startDate = DateUtil.parseStringToDate(model.getBegDate());
}
if (StringUtils.isNotBlank(model.getEndDate())) {
endDate = DateUtil.parseStringToDate(model.getEndDate());
}
public Object userVisitStatAction(ApiUserModel model) {
}
if (startDate != null) {
model.setBegDate(DateUtil.formatToYYYYMMDD(startDate));
}
if (endDate != null) {
model.setEndDate(DateUtil.formatToYYYYMMDD(endDate));
}
long diffInMillies = Math.abs(endDate.getTime() - startDate.getTime());
// 将毫秒转换为天数
long days = diffInMillies / (1000 * 60 * 60 * 24); // 使用long来计算天数
// 只查询正式账号
List<QueryParam> queryParams = Arrays.asList(new QueryParam("isTest", 0));
List<ApiUser> apiUsers = this.apiUserService.findApiUsersByParamsSortDate(queryParams);
// 获取正式账号的ids
List<Integer> ids = new ArrayList<Integer>();
for (ApiUser apiUser : apiUsers) {
ids.add((int) apiUser.getId());
}
// 只获取正式用户的
List<OpenUriAnaly> analyAll = openUriAnalyService.findOpenUriAnalyLogs(model.getUserId(), null, startDate, endDate);
List<OpenUriAnaly> analys = new ArrayList<OpenUriAnaly>();
for (OpenUriAnaly analy : analyAll) {
if (ids.contains(analy.getApiUserId())) {
analys.add(analy);
}
}
List<Integer> userIds = new ArrayList<Integer>();
// 获取有访问记录的用户ID
for (OpenUriAnaly analy : analys) {
if (!userIds.contains(analy.getApiUserId())) {
userIds.add(analy.getApiUserId());
}
}
List<StatAnalyBo> bosAll = new ArrayList<StatAnalyBo>();
if (CollectionUtils.isNotEmpty(userIds)) {
// 根据访问遍历用户
for (Integer userId : userIds) {
long id = (long) userId;
// 获取每个用户的访问
List<OpenUriAnaly> analyUser = openUriAnalyService.findOpenUriAnalyLogs(id, null, startDate, endDate);
ApiUserAuthority authority = apiUserAuthorityService.findByUserId(id);
Map<String, Integer> countMap = authority.getTotalCountMap();
// 获取 deadline
ApiUser apiUser = apiUserService.findApiUserById(id);
// 获取用户名
User user = userService.findById(id);
List<StatAnalyBo> bos = null;
bos = createAnalyBosByLink(analyUser);
long date = DateUtil.parseStringToDate("2016-11-23").getTime();
if (startDate.getTime() >= date && endDate.getTime() > date) {
List<ApiInvokeStatistical> apiInvokeStatisticalList = apiInvokeStatisticalService.findApiInvokeStatisticalList(id, startDate, endDate);
if (bos != null && apiInvokeStatisticalList.size() > 0) {
Map<String, Map<String, Integer>> resultMap = getCountByUri(apiInvokeStatisticalList);
for (StatAnalyBo boTemp : bos) {
boTemp.setPurpose(apiUser.getPurpose());
String url = boTemp.getKey();
boTemp.setSuccessEffectiveTime(resultMap.get(url).get("successEffectiveCount"));
boTemp.setSuccessInvalidTime(resultMap.get(url).get("successInvalidCount"));
boTemp.setDefeatTime(resultMap.get(url).get("defeatCount"));
if (apiUser != null) {
boTemp.setDeadline(apiUser.getDeadTime());
}
long curVistedCount = apiVisitCountService.findApiVisitCountByDay(id, url, DateUtil.truncateDay(new Date()));
long historyCount = apiVisitCountService.findApiVisitAllCount(id, url, DateUtil.truncateDay(new Date()));
if (countMap != null && countMap.containsKey(url)) {
Integer totalCount = countMap.get(url);
long usedCount = curVistedCount + historyCount;
boTemp.setSurplusCallCount(Integer.valueOf((int) (totalCount - usedCount)));
}
if (user != null) {
boTemp.setName(user.getDisplayName());
boTemp.setUserId((int) user.getId());
}
List<ApiVisitCount> apiVisitCounts = apiVisitCountService.findAllApiVisitCount(id, boTemp.getKey(), startDate, endDate);
int total = 0;
for (ApiVisitCount apiVisitCount : apiVisitCounts) {
total += apiVisitCount.getVisitCount();
}
boTemp.setBillingCount(total);
double dailyUse = boTemp.getCount() * 1.0 / days;
boTemp.setDailyUse(dailyUse);
}
}
}
bosAll.addAll(bos);
}
}
this.buildReport(bosAll, response, startDate, endDate);
model.setViewType("stream");
return model;
}
private void buildReport(List<StatAnalyBo> bos, HttpServletResponse response, Date startDate, Date endDate) {
try {
List<List<String>> dataList = new ArrayList<List<String>>();
for (StatAnalyBo bo : bos) {
List<String> row = new ArrayList<String>();
row.add(bo.getUserId() != null ? bo.getUserId().toString() : "");
row.add(bo.getName() != null ? bo.getName().toString() : "");
if (bo.getPurpose() != null) {
Integer purpose = bo.getPurpose();
if (purpose == 1) {
row.add("API");
} else if (purpose == 2) {
row.add("PDS");
} else if (purpose == 4) {
row.add("房生活");
}
}else {
row.add("");
}
row.add(bo.getKey() != null ? bo.getKey().toString() : "");
row.add(bo.getSurplusCallCount() != null ? bo.getSurplusCallCount().toString() : "");
row.add(bo.getDeadline() != null ? DateUtils.format(bo.getDeadline(), "yyyy/MM/dd") : "");
row.add(bo.getCount() != null ? bo.getCount().toString() : "");
if (bo.getDailyUse() !=null){
// 格式化为保留一位小数的字符串
String formattedDailyUse = String.format("%.1f", bo.getDailyUse());
row.add(formattedDailyUse);
}else {
row.add("");
}
row.add(bo.getPercent() != null ? bo.getPercent().toString() : "");
row.add(bo.getSuccessEffectiveTime() != null ? bo.getSuccessEffectiveTime().toString() : "");
row.add(bo.getSuccessInvalidTime() != null ? bo.getSuccessInvalidTime().toString() : "");
row.add(bo.getDefeatTime() != null ? bo.getDefeatTime().toString() : "");
row.add(bo.getBillingCount() != null ? bo.getBillingCount().toString() : "");
dataList.add(row);
}
String fileName = "用户账号使用统计(" + DateUtils.format(startDate, "yyyy/MM/dd") + "—" + DateUtils.format(endDate, "yyyy/MM/dd") + ").xlsx";
// ExcelWriter excelWriter = EasyExcel.write(response.getOutputStream()).build();
String displayName = URLEncoder.encode(fileName, "UTF-8");
response.setHeader("Content-Disposition", "attachment; filename=\"" + displayName + "\"");
response.setContentType("application/x-download;charset=utf-8");
Workbook workbook = new XSSFWorkbook();
Sheet sheet = workbook.createSheet("用户账号使用统计");
List<String> head = Arrays.asList(
"用户ID", "账号", "账号用途", "接口", "剩余使用次数", "合同到期时间", "访问次数", "日均访问量",
"占比(%)", "成功并有效", "成功并无效", "失败次数", "计费次数");
// 创建表头
Row headRow = sheet.createRow(0);
for (int i = 0; i < head.size(); i++) {
Cell cell = headRow.createCell(i);
cell.setCellValue(head.get(i));
}
// 写入数据,从第二行
for (int i = 0; i < dataList.size(); i++) {
Row row = sheet.createRow(i + 1);
List<String> data = dataList.get(i);
for (int j = 0; j < data.size(); j++) {
Cell cell = row.createCell(j);
cell.setCellValue(data.get(j));
}
}
ServletOutputStream outputStream = response.getOutputStream();
workbook.write(outputStream);
} catch (IOException e) {
}
}
List<List<String>> formatHead(String[] headArr) {
List<List<String>> head = new ArrayList<List<String>>();
for (String name : headArr) {
List<String> temp = new ArrayList<String>();
temp.add(name);
head.add(temp);
}
return head;
}
public Object userVisitStatAction(ApiUserModel model) throws IllegalAccessException {
Date startDate = null;
Date endDate = null;
if (StringUtils.isNotBlank(model.getBegDate())) {
@@ -1075,8 +1288,12 @@ public class ApiUserController {
// bos = createAnalyBosByUser(analys);
// model.setChartSeries(toChartString(bos));
// }else if(model.getWay() == 2){
Long userId = model.getUserId();
ApiUserAuthority authority = apiUserAuthorityService.findByUserId(userId);
Map<String, Integer> countMap = authority.getTotalCountMap();
ApiUser apiUser = apiUserService.findApiUserById(userId);
if (model.getWay() == 2) {
User user = userService.findById(model.getUserId());
User user = userService.findById(userId);
if (user != null) {
model.setUserName(user.getDisplayName());
}
@@ -1120,6 +1337,16 @@ public class ApiUserController {
boTemp.setSuccessEffectiveTime(resultMap.get(url).get("successEffectiveCount"));
boTemp.setSuccessInvalidTime(resultMap.get(url).get("successInvalidCount"));
boTemp.setDefeatTime(resultMap.get(url).get("defeatCount"));
if (apiUser != null) {
boTemp.setDeadline(apiUser.getDeadTime());
}
long curVistedCount = apiVisitCountService.findApiVisitCountByDay(userId, url, DateUtil.truncateDay(new Date()));
long historyCount = apiVisitCountService.findApiVisitAllCount(userId, url, DateUtil.truncateDay(new Date()));
if (countMap != null && countMap.containsKey(url)) {
Integer totalCount = countMap.get(url);
long usedCount = curVistedCount + historyCount;
boTemp.setSurplusCallCount(Integer.valueOf((int) (totalCount - usedCount)));
}
}
}
}
@@ -74,6 +74,8 @@ public class InquiryController {
role = fjtRoleService.findRolesByName("农行");
}else if(2 == bkoffUsers.getSupportInquiryTask().intValue()){
role = fjtRoleService.findRolesByName("深圳农行");
}else if(3 == bkoffUsers.getSupportInquiryTask().intValue()){
role = fjtRoleService.findRolesByName("大正评估");
}
List<Long> zqUserIds = userCityService.findUserIdByCity("肇庆");
@@ -87,6 +89,8 @@ public class InquiryController {
}
}else if(vo2.getRole()==16){
userIds.add(vo2.getId());
}else if(vo2.getRole()==18){
userIds.add(vo2.getId());
}
}
@@ -51,6 +51,7 @@
<select name="isTest">
<option value="0">正式账户</option>
<option value="1">测试账户</option>
<option value="2">内部账户</option>
</select>
</td>
</tr>
@@ -12,9 +12,9 @@
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
<style>
label{display:inline-block}
.main-content {
padding-bottom: 0;
}
@@ -47,28 +47,51 @@
</span>
</div>
<form action="<#if way?? && way==0>../apiUser/pandect<#else>../apiUser/apiVisitStat</#if>" method="post">
<input type="hidden" name="way" value="${way!}"/>
<#if way!=3>
<label for="districtName" style="margin-left: 15px;">用户ID:</label>
<input type="text" name="userId" value="${userId!}" class="input-medium" style="margin-top:9px;" ></input>
<#else>
<label for="districtName" style="margin-left: 15px;">城市:</label>
<input type="text" class="input-medium" readonly="readonly" id="city" name="citys" value="${citys!}"/><input type="button" class="input-medium" id="butt" value="选择城市"/>
</#if>
<label for="startDate">&nbsp;&nbsp;&nbsp;&nbsp;日期:</label>
<input placeholder="开始时间" class="form-control" style="width:140px;" name="begDate" id="begDate" value="<#if begDate??>${begDate}</#if>" onfocus="WdatePicker({dateFmt:'yyyy-MM-dd'})" />-
<input id="endDate" placeholder="结束时间" class="form-control" style="width:140px;" name="endDate" value="${endDate!}" onfocus="WdatePicker({dateFmt:'yyyy-MM-dd'})"/>
<form id="myForm" action="<#if way?? && way==0>../apiUser/pandect<#else>../apiUser/apiVisitStat</#if>" method="post">
<input type="hidden" name="way" value="${way!}"/>
<input id="queryBtn" class ="btn btn-primary" type="submit" value="查询" />
<#if way!=3>
<label for="districtName" style="margin-left: 15px;">用户ID:</label>
<input type="text" name="userId" value="${userId!}" class="input-medium" style="margin-top:9px;" />
<#else>
<label for="districtName" style="margin-left: 15px;">城市:</label>
<input type="text" class="input-medium" readonly="readonly" id="city" name="citys" value="${citys!}"/>
<input type="button" class="input-medium" id="butt" value="选择城市"/>
</#if>
<#if way!=3>
<label for="">(备注:成功有效、成功无效、失败的次数统计从2016-11-23开始)</label>
</#if>
</form>
<label for="startDate">&nbsp;&nbsp;&nbsp;&nbsp;日期:</label>
<!-- 下拉菜单选择最近一周、最近一月或自定义 -->
<select id="dateRange" class="form-control" style="width:160px;" onchange="updateDateRange(this.value)">
<option value="custom">自定义</option>
<option value="lastWeek">最近一周</option>
<option value="lastMonth">最近一月</option>
</select>
<!-- 自定义日期输入框 -->
<input placeholder="开始时间" class="form-control" style="width:140px;" name="begDate" id="begDate"
value="<#if begDate??>${begDate}</#if>"
onfocus="WdatePicker({dateFmt:'yyyy-MM-dd'})" />
-
<input id="endDate" placeholder="结束时间" class="form-control" style="width:140px;" name="endDate"
value="${endDate!}"
onfocus="WdatePicker({dateFmt:'yyyy-MM-dd'})" />
<!-- 查询按钮 -->
<input id="queryBtn" class="btn btn-primary" type="submit" value="查询" onclick="submitForm('query')" />
<!-- 下载按钮 -->
<input id="downloadBtn" class="btn btn-secondary" type="submit" value="导出" onclick="submitForm('download')" />
<#if way!=3>
<label for="">(备注:成功有效、成功无效、失败的次数统计从2016-11-23开始)</label>
</#if>
</form>
<#--<input id="expect" class ="btn btn-primary" type="submit" value="导出" />-->
<a class="btn common_btn" href="importExcel?city=${citys!}&begDate=${begDate!}&endDate=${endDate!}">导出</a>
<#-- <a class="btn common_btn" href="importExcel?city=${citys!}&begDate=${begDate!}&endDate=${endDate!}">导出</a>-->
<div id="isHide" style="display: none">
<ul>
@@ -544,7 +567,7 @@
</tr>
</thead>
<tbody>
<#if statAnalyBoList??>
<#if statAnalyBoList??>
<#list statAnalyBoList as bo>
<tr>
<#if way?? && way==1>
@@ -591,7 +614,7 @@
<#else>
<tr><td ></td></tr>
</#if>
</tbody>
</table>
<div class="tabbable" style="padding-top: 60px;">
@@ -643,7 +666,7 @@
$("#statBut3").removeClass("cur");
});
$("#statBut2").click(function(){
this.addClass("cur");
$("#statBut1").removeClass("cur");
@@ -697,8 +720,8 @@
}
})
});
$(function () {
$('#container').highcharts({
@@ -734,8 +757,46 @@
});
});
</script>
function updateDateRange(option) {
const now = new Date();
const begDateInput = document.getElementById('begDate');
const endDateInput = document.getElementById('endDate');
if (option === 'lastWeek') {
const lastWeek = new Date();
lastWeek.setDate(now.getDate() - 7);
begDateInput.value = formatDate(lastWeek);
endDateInput.value = formatDate(now);
} else if (option === 'lastMonth') {
const lastMonth = new Date();
lastMonth.setMonth(now.getMonth() - 1);
begDateInput.value = formatDate(lastMonth);
endDateInput.value = formatDate(now);
} else {
begDateInput.value = '';
endDateInput.value = '';
}
}
function formatDate(date) {
const year = date.getFullYear();
const month = String(date.getMonth() + 1).padStart(2, '0');
const day = String(date.getDate()).padStart(2, '0');
return year + '-' + month + '-' + day;
}
function submitForm(action) {
const form = document.getElementById('myForm');
if (action === 'query') {
debugger
form.action = "<#if way?? && way==0>../apiUser/pandect<#else>../apiUser/apiVisitStat</#if>";
} else if (action === 'download') {
form.action = "../apiUser/export";
}
form.submit();
}
</script>
</body>
</html>
</#compress>
@@ -61,12 +61,18 @@
<td>账户类型</td>
<td>
<select name="isTest">
<#if apiUserBo.isTest==0>
<#if apiUserBo.isTest == 0>
<option value="0" selected="selected">正式账户</option>
<option value="1">测试账户</option>
<#else>
<option value="0"> 正式账户</option>
<option value="2">内部账户</option>
<#elseif apiUserBo.isTest == 1>
<option value="0">正式账户</option>
<option value="1" selected="selected">测试账户</option>
<option value="2">内部账户</option>
<#else>
<option value="0">正式账户</option>
<option value="1">测试账户</option>
<option value="2" selected="selected">内部账户</option>
</#if>
</select>
</td>
@@ -131,8 +131,10 @@
<td align="center">
<#if bo.isTest==0>
正式账户
<#else>
<#elseif bo.isTest==1>
测试账户
<#else>
内部账户
</#if>
</td>
<td>
@@ -91,6 +91,12 @@ table tr th {
<#if query=="uri">URL<#elseif query=="day">日期<#else>访问时长</#if>
</#if>
</th>
<th width="9%" colspan="1">
剩余使用次数
</th>
<th width="9%" colspan="1">
合同到期时间
</th>
<th width="9%" colspan="1">
访问次数
</th>
@@ -145,6 +151,8 @@ table tr th {
</#if>
</#if>
</td>
<td width="9%"><#if bo.surplusCallCount??>${bo.surplusCallCount}</#if></td>
<td width="9%"><#if bo.deadline??>${bo.deadline?string("yyyy-MM-dd")}</#if></td>
<td width="9%">${bo.count}</td>
<td width="9%">${bo.percent}</td>
<#if way==2 && query=='uri'><td>${resourceMap[bo.key]!}</td></#if>
@@ -176,7 +176,7 @@ table{text-align: left;}
}
})
if(role==12||role==17){
if(role==12||role==17||role==18){
if(userStr.length>0){
console.log(userStr);
$("input[name=roleUsers]").val(userStr);
@@ -306,7 +306,7 @@ $("input[name=email]").blur(function(){
$("#role").change(function() {
var role = $(this).val();
//只对农行生效
if (role == 12 || role == 17) {
if (role == 12 || role == 17 || role == 18) {
$("#show_chk_all").removeAttr("style");
jQuery.ajax({
type: 'POST',
@@ -340,7 +340,7 @@ $("input[name=email]").blur(function(){
}
return false;
}
<#if user?? && (user.role == 12 || user.role == 17)>
<#if user?? && (user.role == 12 || user.role == 17|| user.role == 18)>
$("#show_chk_all").removeAttr("style");
jQuery.ajax({
type: 'POST',
@@ -7,7 +7,7 @@ public interface Constants {
String FANGJIA_CRAWL_MONGO_URI = "mongodb://192.168.0.235:27018";
String LOG_MONGO_URI = "mongodb://192.168.0.42:52002";
String SEABIRD_MONGO_URI = "mongodb://192.168.0.74:57007";
String SEAWATER_MONGO_URI = "mongodb://192.168.0.206:27020";
String SEAWATER_MONGO_URI = "mongodb://root:Goojia%40206!!#@192.168.0.206:27020";
String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.100:38080/FangJiaDataService";
}
@@ -15,15 +15,15 @@ import com.fangjia.dal.MongoEntity;
public class SqmetreRent implements MongoEntity {
/**
*
*
*/
private static final long serialVersionUID = -5525712882803653336L;
private String id;
private String city;
private String region;
private String districtName;
private Integer avgPrice;
private Integer calcAvgPrice;
private Number avgPrice;
private Number calcAvgPrice;
private Double markup;
private Date weekend;
private String cat;
@@ -74,20 +74,20 @@ public class SqmetreRent implements MongoEntity {
}
@Column(name = "avg_price")
public Integer getAvgPrice() {
public Number getAvgPrice() {
return avgPrice;
}
public void setAvgPrice(Integer avgPrice) {
public void setAvgPrice(Number avgPrice) {
this.avgPrice = avgPrice;
}
@Column(name = "calc_avg_price")
public Integer getCalcAvgPrice() {
public Number getCalcAvgPrice() {
return calcAvgPrice;
}
public void setCalcAvgPrice(Integer calcAvgPrice) {
public void setCalcAvgPrice(Number calcAvgPrice) {
this.calcAvgPrice = calcAvgPrice;
}
@@ -163,7 +163,5 @@ public class SqmetreRent implements MongoEntity {
public void setCat(String cat) {
this.cat = cat;
}
}
@@ -26,7 +26,7 @@ public class ApiUser implements SqlEntity {
private Date creationDate;
private Date modificationDate;
private String company;
private int isTest; //0-正式客戶 1-测试用户
private int isTest; //0-正式客戶 1-测试用户 2-内部测试用户
private Date deadTime;
/**
* 账户用途
@@ -32,6 +32,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
import org.apache.http.util.EntityUtils;
import org.sothis.core.util.CollectionUtils;
import org.sothis.core.util.Pager;
import sun.misc.BASE64Encoder;
@@ -42,6 +43,7 @@ import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.nio.charset.Charset;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
@@ -439,6 +441,8 @@ public class AddrSvcController {
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else if (!"2".equals(type)) { // 对私的不用齐梁
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
@@ -450,6 +454,12 @@ public class AddrSvcController {
Map<String, String> map = senRequest(address, qltype);
// 涉农结果不一样
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
// 齐梁县政府但是涉农为农,取我们的结果 扬中市三茅街道中华村永固河路1号
if ("是".equals(map.get("countyGov")) && "农".equals(map.get("verifyFlag"))) {
basicModel.setResult(result);
mav.setModel(basicModel);
return mav;
}
//城市不一样 用我们
if (result.containsKey("city")) {
if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
@@ -457,11 +467,15 @@ public class AddrSvcController {
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("typeName", areaMap.get(map.get("typeCode")));
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
basicModel.setResult(map);
}
@@ -488,6 +502,45 @@ public class AddrSvcController {
}
return mav;
}
/**
* 判断县级市
*/
public String countyLevelCity(Map<String, String> map) {
if (map.get("city") == null) {
return "";
}
if (map.get("city").endsWith("区")) {
return "否";
}
if (map.get("city").equals("东莞市") || map.get("city").equals("嘉峪关市") || map.get("city").equals("中山市") || map.get("city").equals("儋州市") ) {
return "否";
}
return "是";
}
/**
* 判断所属区域
*
* @param map
* @return
*/
public String addrArea(Map<String, String> map, String type) {
if (map.get("verifyFlag") == null ) {
return "";
}
if (map.get("verifyFlag").equals("非农")) {
return "城市区域";
}
if (map.get("verifyFlag").equals("农")) {
return "农村区域";
}
if ("2".equals(type) && "是".equals(map.get("countyGov"))) {
return "县城城区";
}
return "";
}
/**
* 涉农识别-新规则-上海征信
*
@@ -530,7 +583,8 @@ public class AddrSvcController {
//我们省级及以上有开发区,直接用我们的结果
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else if (!"2".equals(type)) { // 对私的不用齐梁
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
@@ -542,17 +596,27 @@ public class AddrSvcController {
Map<String, String> map = senRequest(address, qltype);
// 涉农结果不一样
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
// 齐梁县政府但是涉农为农,取我们的结果 扬中市三茅街道中华村永固河路1号
if ("是".equals(map.get("countyGov")) && "农".equals(map.get("verifyFlag"))) {
basicModel.setResult(result);
mav.setModel(basicModel);
return mav;
}
//城市不一样 用我们
if (result.containsKey("city")) {
if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
basicModel.setResult(result);
} else {
map.put("addrVerify", "地址有效");
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
@@ -607,7 +671,8 @@ public class AddrSvcController {
params.put("type", type);
params.put("showFullValue", "2");
Pager pager = new Pager(1, 1);
VillageIdentify villageResult = villageIdentifyService.findAll(villageType, model.getCity(), model.getAddress(), pager).get(0);
List<VillageIdentify> villageIdentifyList = villageIdentifyService.findByCityAndAddress(villageType, model.getCity(), model.getAddress(), pager);
VillageIdentify villageResult = CollectionUtils.isNotEmpty(villageIdentifyList) ? villageIdentifyList.get(0) : null;
if (villageResult != null && StringUtils.isNotBlank(villageResult.getVerifyFlag())) {
Map<String, String> resultMap = new HashMap<String, String>();
resultMap.put("province",villageResult.getProvince());
@@ -659,7 +724,8 @@ public class AddrSvcController {
//我们省级及以上有开发区,直接用我们的结果
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else if (!"2".equals(type)) { // 对私的不用齐梁
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
@@ -672,17 +738,27 @@ public class AddrSvcController {
Map<String, String> map = senRequest(address, qltype);
// 涉农结果不一样
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
// 齐梁县政府但是涉农为农,取我们的结果 扬中市三茅街道中华村永固河路1号
if ("是".equals(map.get("countyGov")) && "农".equals(map.get("verifyFlag"))) {
basicModel.setResult(result);
mav.setModel(basicModel);
return mav;
}
//有市辖建制镇, 或者城市不一样 用我们
if (result.containsKey("city")) {
if ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
basicModel.setResult(result);
} else {
map.put("addrVerify", "地址有效");
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
} else {
map.put("addrVerify", "地址有效");
map.put("addrArea", addrArea(map, type));
map.put("countyLevelCity", countyLevelCity(map));
map.put("typeName", areaMap.get(map.get("typeCode")));
basicModel.setResult(map);
}
File diff suppressed because it is too large Load Diff
@@ -203,7 +203,7 @@ public class RentController {
private Double getSqmetreRent(District district){
SqmetreRent sqmetreRent = sqmetreRentService.findNearestSqmetreRent(district.getCity(), district.getRegion(), district.getName(), null, "district");
if(null != sqmetreRent){
return Double.valueOf(sqmetreRent.getAvgPrice());
return Double.valueOf(sqmetreRent.getAvgPrice().intValue());
}
return null;
}
@@ -1,24 +1,32 @@
package com.fangjia.open.controllers;
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.*;
import com.fangjia.core.util.AnalyticsUtil;
import com.fangjia.core.util.BeanUtils;
import com.fangjia.core.util.DateUtil;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.mongo.model.SendSmsLog;
import com.fangjia.dal.mongo.open.model.ApiUserAuthority;
import com.fangjia.dal.mongo.open.model.OpenResource;
import com.fangjia.dal.mongo.tool.model.OpenUriAnaly;
import com.fangjia.open.HttpStatusCodeEnum;
import com.fangjia.open.interceptors.AuthenticationBean;
import com.fangjia.open.model.*;
import com.fangjia.open.model.weixin.SendCodeJsonModel;
import com.fangjia.open.model.weixin.SendCodeModel;
import com.fangjia.service.open.*;
import net.sf.json.JSONObject;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang.StringUtils;
import org.sothis.web.mvc.ActionContext;
import org.sothis.web.mvc.ActionInvocation;
import org.sothis.web.mvc.annotation.Param;
import com.fangjia.dal.sql.model.User;
import com.fangjia.open.HttpCode;
import com.fangjia.open.OpenHttpCodeEnum;
import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.open.model.UsersModel;
import com.fangjia.service.mongo.mobile.UserInfoService;
import com.fangjia.service.sql.UserService;
import com.fangjia.service.util.UsersUtils;
@@ -26,100 +34,170 @@ import com.fangjia.service.util.UsersUtils;
import javax.servlet.http.HttpServletRequest;
public class UsersController {
private UserService userService;
private UserInfoService userInfoService;
private UserService userService;
private UserInfoService userInfoService;
/**
*
* @param model
* @return 1 无此用户 2 非法请求 3 密码错误 0 正常登陆
* @throws Exception
*/
public Object loginAction(UsersModel model) throws Exception {
Map<String, Object> infomain = new HashMap<String, Object>();
if (StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getPwd())) {
infomain.put("flag", UsersUtils.LoginEnum.EINVAL.ordinal());
} else {
User user = userService.findByUserName(model.getName());
if (null == user) {
infomain.put("flag", UsersUtils.LoginEnum.NOUSER.ordinal());
} else if (!user.getPassword().equals(DigestUtils.md5Hex(model.getPwd()))) {
infomain.put("flag", UsersUtils.LoginEnum.ERRPWD.ordinal());
} else {
infomain.put("flag", UsersUtils.LoginEnum.SUCCESS.ordinal());
infomain.put("displayname", user.getDisplayName());
infomain.put("username", user.getUserName());
infomain.put("id", user.getId());
}
}
JSONObject json = JSONObject.fromObject(infomain);
return json.toString();
}
/**
*
* @param model
* @return 1 无此用户 2 非法请求 3 原始密码错误 0 修改成功
* @throws Exception
*/
public Object modifyPwdAction(UsersModel model) throws Exception {
Map<String, Object> infomain = new HashMap<String, Object>();
if (StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getPwd())) {
infomain.put("flag", UsersUtils.LoginEnum.EINVAL.ordinal());
} else {
User user = userService.findByUserName(model.getName());
if (null == user) {
infomain.put("flag", UsersUtils.LoginEnum.NOUSER.ordinal());
} else if (!user.getPassword().equals(DigestUtils.md5Hex(model.getPwd()))) {
infomain.put("flag", UsersUtils.LoginEnum.ERRPWD.ordinal());
} else {
user.setPassword(DigestUtils.md5Hex(model.getNewPwd()));
userService.updateUser(user);
infomain.put("flag", UsersUtils.LoginEnum.SUCCESS.ordinal());
}
}
JSONObject json = JSONObject.fromObject(infomain);
return json.toString();
}
public Object saveAction(@Param(name = "ime") String ime, @Param(name = "name") String name, @Param(name = "description") String description, @Param(name = "image") String image, @Param(name = "type") String type) {
if (ime == null || "".equals(ime.trim())) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "手机唯一标示缺失!"));
return mav;
}
if (name == null || "".equals(name.trim())) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "用户名缺失!"));
return mav;
}
if (type == null || "".equals(type.trim())) {
type = "default";
}
com.fangjia.dal.mongo.mobile.model.User userInfo = userInfoService.save(ime, name, description, image, type);
Map<String, Object> infomain = new HashMap<String, Object>();
infomain.put("uid", userInfo.getId());
infomain.put("ime", userInfo.getIme());
infomain.put("name", userInfo.getName());
infomain.put("type", userInfo.getType());
infomain.put("description", userInfo.getDescription());
infomain.put("image", userInfo.getImage());
JSONObject json = JSONObject.fromObject(infomain);
return json.toString();
}
public void setUserService(UserService userService) {
this.userService = userService;
}
public void setUserInfoService(UserInfoService userInfoService) {
this.userInfoService = userInfoService;
}
private ApiVisitCountService apiVisitCountService;
private ApiUserAuthorityService apiUserAuthorityService;
private OpenResourceService openResourceService;
private OpenUriAnalyService openUriAnalyService;
/**
* @param model
* @return 1 无此用户 2 非法请求 3 密码错误 0 正常登陆
* @throws Exception
*/
public Object loginAction(UsersModel model) throws Exception {
Map<String, Object> infomain = new HashMap<String, Object>();
if (StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getPwd())) {
infomain.put("flag", UsersUtils.LoginEnum.EINVAL.ordinal());
} else {
User user = userService.findByUserName(model.getName());
if (null == user) {
infomain.put("flag", UsersUtils.LoginEnum.NOUSER.ordinal());
} else if (!user.getPassword().equals(DigestUtils.md5Hex(model.getPwd()))) {
infomain.put("flag", UsersUtils.LoginEnum.ERRPWD.ordinal());
} else {
infomain.put("flag", UsersUtils.LoginEnum.SUCCESS.ordinal());
infomain.put("displayname", user.getDisplayName());
infomain.put("username", user.getUserName());
infomain.put("id", user.getId());
}
}
JSONObject json = JSONObject.fromObject(infomain);
return json.toString();
}
/**
* @param model
* @return 1 无此用户 2 非法请求 3 原始密码错误 0 修改成功
* @throws Exception
*/
public Object modifyPwdAction(UsersModel model) throws Exception {
Map<String, Object> infomain = new HashMap<String, Object>();
if (StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getPwd())) {
infomain.put("flag", UsersUtils.LoginEnum.EINVAL.ordinal());
} else {
User user = userService.findByUserName(model.getName());
if (null == user) {
infomain.put("flag", UsersUtils.LoginEnum.NOUSER.ordinal());
} else if (!user.getPassword().equals(DigestUtils.md5Hex(model.getPwd()))) {
infomain.put("flag", UsersUtils.LoginEnum.ERRPWD.ordinal());
} else {
user.setPassword(DigestUtils.md5Hex(model.getNewPwd()));
userService.updateUser(user);
infomain.put("flag", UsersUtils.LoginEnum.SUCCESS.ordinal());
}
}
JSONObject json = JSONObject.fromObject(infomain);
return json.toString();
}
public Object saveAction(@Param(name = "ime") String ime, @Param(name = "name") String name, @Param(name = "description") String description, @Param(name = "image") String image, @Param(name = "type") String type) {
if (ime == null || "".equals(ime.trim())) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "手机唯一标示缺失!"));
return mav;
}
if (name == null || "".equals(name.trim())) {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "用户名缺失!"));
return mav;
}
if (type == null || "".equals(type.trim())) {
type = "default";
}
com.fangjia.dal.mongo.mobile.model.User userInfo = userInfoService.save(ime, name, description, image, type);
Map<String, Object> infomain = new HashMap<String, Object>();
infomain.put("uid", userInfo.getId());
infomain.put("ime", userInfo.getIme());
infomain.put("name", userInfo.getName());
infomain.put("type", userInfo.getType());
infomain.put("description", userInfo.getDescription());
infomain.put("image", userInfo.getImage());
JSONObject json = JSONObject.fromObject(infomain);
return json.toString();
}
public Object surplusCallCountAction() throws IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
ActionContext context = ActionContext.getContext();
HttpServletRequest request = context.getRequest();
AuthenticationBean instance = (AuthenticationBean) request.getAttribute(AuthenticationBean.ATTR_AUTHENTICATION_BEAN);
if (null != instance) {
User user = instance.user;
List<OpenResource> openResourceList = openResourceService.findAll();
List<ApiMonitorInfo> list = new ArrayList<ApiMonitorInfo>();
long id = user.getId();
ApiUserAuthority authority = apiUserAuthorityService.findByUserId(id);
if (null != authority) {
// 用户有的uri
List<OpenResource> userResources = new ArrayList<OpenResource>();
for (OpenResource openResource : openResourceList) {
if (authority.getOpenResourseUris().contains(openResource.getUri())) {
userResources.add(openResource);
}
}
Map<String, Integer> countMap = authority.getTotalCountMap();
if (null != countMap) {
// 改成遍历用户有的uri
for (OpenResource openResource : userResources) {
String uri = openResource.getUri();
ApiMonitorInfo info = new ApiMonitorInfo();
info.setApiUri(uri);
info.setApiName(openResource.getName());
info.setUserId((int) id);
info.setUsername(user.getUserName());
if (null != countMap.get(uri) && countMap.get(uri) > 0) {
long curVistedCount = apiVisitCountService.findApiVisitCountByDay(id, uri, DateUtil.truncateDay(new Date()));
long historyCount = apiVisitCountService.findApiVisitAllCount(id, uri, DateUtil.truncateDay(new Date()));
Integer totalCount = countMap.get(uri);
info.setTotalCallCount(Long.valueOf(totalCount));
long usedCount = curVistedCount + historyCount;
info.setSurplusCallCount(totalCount - usedCount);
info.setUsedCallCount(usedCount);
}
list.add(info);
}
}
}
BasicModel basicModel = new BasicModel(200, "ok");
basicModel.setResult(list);
mav.setModel(basicModel);
} else {
mav.setModel(new BasicModel(HttpStatusCodeEnum.BAD_REQUEST_ERROR_KEY));
}
return mav;
}
public void setUserService(UserService userService) {
this.userService = userService;
}
public void setUserInfoService(UserInfoService userInfoService) {
this.userInfoService = userInfoService;
}
public void setApiVisitCountService(ApiVisitCountService apiVisitCountService) {
this.apiVisitCountService = apiVisitCountService;
}
public void setApiUserAuthorityService(ApiUserAuthorityService apiUserAuthorityService) {
this.apiUserAuthorityService = apiUserAuthorityService;
}
public void setOpenResourceService(OpenResourceService openResourceService) {
this.openResourceService = openResourceService;
}
public void setOpenUriAnalyService(OpenUriAnalyService openUriAnalyService) {
this.openUriAnalyService = openUriAnalyService;
}
}
@@ -0,0 +1,81 @@
package com.fangjia.open.model;
import java.util.Date;
public class ApiMonitorInfo {
private Integer userId;
private String username;
private Integer callCount;
private Date callTime;
private String apiUri;
private String apiName;
private Long surplusCallCount;
private Long totalCallCount;
private Long usedCallCount;
private Date endTime;
public Long getSurplusCallCount() {
return surplusCallCount;
}
public void setSurplusCallCount(Long surplusCallCount) {
this.surplusCallCount = surplusCallCount;
}
public Date getEndTime() {
return endTime;
}
public void setEndTime(Date endTime) {
this.endTime = endTime;
}
public Integer getUserId() {
return userId;
}
public void setUserId(Integer userId) {
this.userId = userId;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public Integer getCallCount() {
return callCount;
}
public void setCallCount(Integer callCount) {
this.callCount = callCount;
}
public Date getCallTime() {
return callTime;
}
public void setCallTime(Date callTime) {
this.callTime = callTime;
}
public String getApiUri() {
return apiUri;
}
public void setApiUri(String apiUri) {
this.apiUri = apiUri;
}
public String getApiName() {
return apiName;
}
public void setApiName(String apiName) {
this.apiName = apiName;
}
public Long getTotalCallCount() {
return totalCallCount;
}
public void setTotalCallCount(Long totalCallCount) {
this.totalCallCount = totalCallCount;
}
public Long getUsedCallCount() {
return usedCallCount;
}
public void setUsedCallCount(Long usedCallCount) {
this.usedCallCount = usedCallCount;
}
}
@@ -27,6 +27,15 @@ public class AvgPriceQueryModel {
Integer periodCount;
String periodType;
Integer houseType;
String cat;
public String getCat() {
return cat;
}
public void setCat(String cat) {
this.cat = cat;
}
public Integer getPeriodCount() {
return periodCount;
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -0,0 +1,40 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
*/
class AddThirdPriceSample {
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new AddThirdPriceSample()
task.run()
}
def run(){
for(def i=1;i<19;i++){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
if(d['A'] && d['C']){
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
if(dist){
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger())
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
}else{
println d['A'] +" 没查到"
}
}
}, i)
println(i)
}
}
}
@@ -34,13 +34,15 @@ class BigdataCommunityBatch extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def fangjiaDists = new HashSet()
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
}
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -52,14 +54,18 @@ class BigdataCommunityBatch extends BaseBatch{
def today = new Date().getDate()
println "starting 。。。。。。。。。"
def allDists = CcbCommonUtils.findAllDistrictByCity(null, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
for(def d : allDists){
fangjiaDists << d._id.toString()
}
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
println "加载建行小区量"
def ccbMaps = null
if(today<=10){
@@ -67,11 +73,11 @@ class BigdataCommunityBatch extends BaseBatch{
}else{
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
}
println "组装新数据"
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()+" 结果:"+rows.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
@@ -103,10 +109,7 @@ class BigdataCommunityBatch extends BaseBatch{
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
@@ -70,12 +70,12 @@ class BigdataDealBatch extends BaseBatch{
//664b73800000000000000000 5-21
def citys = CcbCommonUtils.getRunningCitys(args)
// println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
// addDealPriceMatch(city);
// }
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
def rows = []
@@ -20,7 +20,7 @@ import java.nio.charset.StandardCharsets
class CcbCommonUtils {
public static def citys_fjw_all = ["上海","长春","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_fjw_all = ["长春","上海","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
@@ -442,9 +442,12 @@ class CcbCommonUtils {
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
if(city){
q.put("city", city);
}
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","荣和怡景园")
// q.put("name","一米阳光一期")
// q.put("name_cat", 0);
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
@@ -456,9 +459,9 @@ class CcbCommonUtils {
}
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
def rets = new HashSet()
def ids = new HashSet()
def ids = []
for(def d : dbsour){
if(kind){
if(d.address){
@@ -688,6 +691,8 @@ class CcbCommonUtils {
pgsqlInstance.executeUpdate(sql2)
}
public static def findGuiDingIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
@@ -199,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
}
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
// if(d['G']){
// def obj = CommonDatautils.findDistrictById(d['G'])
// if(obj){
// d['H'] = obj.name
// }
// }
//})
////def url = "https://sh.anjuke.com"
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
@@ -602,9 +612,9 @@ def findAllData(sqlInstance236, citycode){
////},0)
//
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d->
// if(d['A']){
// def dbo = CommonDatautils.findDistrictById(d['D'])
// def dbo = CommonDatautils.findDistrictById(d['A'])
// if(dbo){
// def udata = new BasicDBObject()
//// udata.put("name_cat", 11)
@@ -620,6 +630,38 @@ def findAllData(sqlInstance236, citycode){
// }
//},0)
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
if(d['A'] && d['B']){
def dbo = CommonDatautils.findDistrictById(d['A'])
if(dbo){
def alias = null
if(!dbo.alias){
alias = []
}else{
alias = dbo.alias
}
if(!alias.contains(d['B'])){
alias << d['B']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("m_date",new Date())
def q = new BasicDBObject()
q.put("_id", dbo._id)
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}else{
println d['A']+" "+"别名已存在"
}
}else{
println dbo._id.toString() +" 没找到"
}
}
})
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
//for(def c : citys){
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
@@ -1289,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240626人工指导价.xlsx"
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['J']){
// def region = RegionData.formatRegion(d['A'], d['J'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
//
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['J']){
//// def region = RegionData.formatRegion(d['A'], d['J'])
//// if(region){
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['L'] = dist.status-dist.name_cat
//// }
//// }
//// if(d['J'] instanceof String){
// def dist = CommonDatautils.findDistrictById(d['J'])
// if(dist){
// d['A'] = dist.city
// d['B'] = dist.region
// d['C'] = dist.name
// d['L'] = dist.status-dist.name_cat
// }
// }
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
// d['K'] = dist.estate_type2
// d['L'] = dist.status
// d['M'] = dist.name_cat
// d['H'] = dist.name
}
// }
}
},0)
//
//// d['H'] = dist.name
// }
//// }
// }
//},0)
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
@@ -1352,24 +1394,57 @@ ExcelUtils.multiThreadingModify(fileName, {d->
//
//})
//def filePath = "D:\\数据\\建行\\房产估值\\指导价\\大庆删除小区.xlsx"
//
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
//ExcelUtils.modify(filePath, { d ->
// if(d['A']){
//
// def obj = CommonDatautils.findDistrictById(d['A'])
// if(obj){
//
// def data = new BasicDBObject()
// data.put("status", 1)
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])), new BasicDBObject('$set', data))
//// def obj = CommonDatautils.findDistrictById(d['A'])
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
// if(list){
// if(list.size()==1){
// def dbo = list[0]
// if(dbo){
// def data = new BasicDBObject()
// data.put("region", d['B'])
// data.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
// }
// }else{
// println d['C'] +" 有多个"
// }
// }
//// if(obj){
//// def lng = d['G']
//// def lat =d['H']
////
//// // 高德转百度
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
//// def lng2 = baidu.x
//// def lat2 = baidu.y
////
////
//// def data = new BasicDBObject()
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
//// data.put("lng", lng_new)
//// data.put("lat", lat_new)
////
//// Long dLat2 = lat2 * 10000000000L
//// Long dLng2 = lng2 * 10000000000L
//// data.put("lat2", dLat2)
//// data.put("lng2", dLng2)
////
////// data.put("status", 1)
//// data.put("m_date", new Date())
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
//// }
// }
//
//})
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// if(d['B'] && !d['D']){
@@ -0,0 +1,14 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
if(d['G'] && d['G'] instanceof String){
def obj = CommonDatautils.findDistrictById(d['G'])
if(obj){
d['H'] = obj.name
}
}
})
@@ -257,6 +257,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -303,6 +306,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
web = "·¿¶à¶à"
}else if(k=="wiwj_avg_price"){
web = "ÎÒ°®ÎÒ¼Ò"
}else if(k=="fccs_avg_price"){
web = "·¿²ú³¬ÊÐ"
}
ret << sourceMap.get(web)
@@ -584,6 +589,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -32,6 +32,10 @@ class TradeStatLog {
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
@@ -41,7 +45,9 @@ class TradeStatLog {
static main(args) {
def task = new TradeStatLog()
// task.run()//统计最近3个月抓取的成交数据量
task.exportTrade()//导出未入库成交
// task.exportTrade()//导出未入库成交
task.exportTrade2()//导出未入库成交
// task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
// task.averageStat()
@@ -584,6 +590,36 @@ class TradeStatLog {
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
}
def exportTrade2(){
// def startDate = "2024-06-01"
// def endDate = "2024-07-01"
def ids = findAllDealIds();
println ids.size()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
if(ids.contains(d['A'])){
d['V'] = "供"
}
if(d['S'] && !d['W']){
def dist = CommonDatautils.findDistrictById(d['S'])
if(!dist.name_cat && dist.name_cat!=0){
d['W'] = "name_cat过滤"
}
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
d['W'] = "小区过滤"
}
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
def map = findDealByCommId(d['B'], d['S'])
def count = map.get(key)
if(count>=2){
d['W'] = "重复"
}
}
println d['A']
})
}
def deleteDistrict(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
if(d['G']){
@@ -637,11 +673,39 @@ class TradeStatLog {
return trades
}
def findAllDealIds(){
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
def ids = []
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.gujia_bds_mdl_id)){
ids << r.gujia_bds_mdl_id
}
})
return ids
}
def findDealByCommId(city, commid){
def q = new BasicDBObject()
q.put("match_id", commid)
q.put("city", city)
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
def dbcursor = DEAL_PRICE_COLL.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def count = map.get(key)
if(!count){
count = 0
}
map.put(key, count+1)
}
return map
}
def findDealIds(city, startDate, endDate){
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
def ids = []
def detailInfos = []
sqlInstance.eachRow(sql,{r->
pssqlInstance.eachRow(sql,{r->
if(!ids.contains(r.dealid)){
ids << r.dealid
@@ -146,7 +146,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommunityGJRH(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
@@ -221,7 +221,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
@@ -253,7 +253,7 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
}
@@ -295,7 +295,7 @@ class ExportDistrictRongHe extends BaseBatch{
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
}
@@ -327,7 +327,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
@@ -356,7 +356,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
@@ -384,7 +384,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
@@ -422,7 +422,7 @@ class ExportDistrictRongHe extends BaseBatch{
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
@@ -452,7 +452,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
@@ -485,7 +485,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
@@ -517,7 +517,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
@@ -552,7 +552,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
@@ -576,7 +576,7 @@ class ExportDistrictRongHe extends BaseBatch{
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
@@ -676,7 +676,7 @@ class ExportDistrictRongHe extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -716,7 +716,7 @@ class ExportDistrictRongHe extends BaseBatch{
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -987,7 +987,7 @@ class ExportDistrictRongHe extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
@@ -998,7 +998,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataBuildingFJW(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
@@ -1072,7 +1072,7 @@ class ExportDistrictRongHe extends BaseBatch{
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
@@ -1084,7 +1084,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportThirdPartyPrice2(price_date, new_dists) {
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1132,7 +1132,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgThirdPartyPrice2(price_date, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1215,7 +1215,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
@@ -1261,7 +1261,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
@@ -1304,7 +1304,7 @@ class ExportDistrictRongHe extends BaseBatch{
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1319,7 +1319,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1393,7 +1393,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
@@ -1460,7 +1460,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1501,7 +1501,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1537,7 +1537,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1570,7 +1570,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
@@ -1609,7 +1609,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
@@ -1652,7 +1652,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
@@ -1689,7 +1689,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1737,7 +1737,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataDistPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1784,7 +1784,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1845,7 +1845,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
@@ -1856,7 +1856,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportBigdataForsale(publishDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
@@ -1913,7 +1913,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportPgBigdataForsale(publishDate,endDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
@@ -2098,7 +2098,7 @@ class ExportDistrictRongHe extends BaseBatch{
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
def dataCodeFileName = "/home/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
@@ -1209,6 +1209,13 @@ class CommonDatautils {
return rets;
}
public static def findAllCount(String cat){
DBObject q = new BasicDBObject();
q.put("cat", cat);
q.put("status", 0)
return SEAWEED_MONGO_COLL.count(q)
}
public static def findDistrictsByName(String city, String name, String cat){
@@ -4653,6 +4660,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","区域")
q.put("district_diff",1)
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4662,7 +4670,7 @@ class CommonDatautils {
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
keys.put("district_short",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def regionMap = [:]
for(def d : dbcursor){
if(d.mzb_district_name){
@@ -4686,7 +4694,7 @@ class CommonDatautils {
def q = new BasicDBObject()
q.put("type","乡镇街道")
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
// q.put("city","芜湖市")
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
def keys = new BasicDBObject()
keys.put("province",1)
@@ -4712,10 +4720,42 @@ class CommonDatautils {
}
static def queryTownByCity(city){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",city)
def keys = new BasicDBObject()
keys.put("province",1)
keys.put("city",1)
keys.put("district",1)
keys.put("street",1)
keys.put("shape",1)
keys.put("alias",1)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def townMap = [:]
for(def d : dbcursor){
def key = d.district
def list = townMap.get(key)
if(!list){
list = []
}
list << d
townMap.put(key, list)
}
println "乡镇街道 加载完成"
return townMap
}
static def queryNewTownByCity(){
def q = new BasicDBObject()
q.put("type","乡镇街道")
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
def keys = new BasicDBObject()
keys.put("city",1)
@@ -4833,6 +4873,9 @@ class CommonDatautils {
def jsonObject = null
def stop = false
def count = 0
def road = ""
def name = ""
def type = ""
while(!stop){
if(glocation){
jsonObject = GaoDeApiUtils.geocode(glocation)
@@ -4852,8 +4895,26 @@ class CommonDatautils {
}
format_addr = jsonObject.regeocode.formatted_address
if(addressComponent.streetNumber){
if(addressComponent.streetNumber.street){
road = addressComponent.streetNumber.street
}
if(addressComponent.streetNumber.number){
road = road + addressComponent.streetNumber.number
}
}
if(addressComponent.building){
if(addressComponent.building.name){
name = addressComponent.building.name
}
if(addressComponent.building.type){
type = addressComponent.building.type
}
}
format_addr = jsonObject.regeocode.formatted_address
}
}
}
@@ -4871,6 +4932,9 @@ class CommonDatautils {
map.district = district
map.township = township
map.format_addr = format_addr
map.road = road
map.type = type
map.name = name
return map;
}
@@ -0,0 +1,79 @@
package com.fangjia.script.gj
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class GreenBuild {
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new GreenBuild()
task.run()
}
def run(){
def datas = findAll()
def rets = match(datas)
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
}
def match(datas){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
for(def i=0;i<datas.size();i++){
final def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
data.match_id = resultMap.get("id").toString()
data.match_city = resultMap.get("city")
data.match_district = resultMap.get("district")
data.match_name = resultMap.get("name")
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
}else{
rets << [data._id.toString(), data.city, data.district, data.name]
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
println "查询完成......."
latch.await();
return rets
}
def findAll(){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("district", 1)
keys.put("name", 1)
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
}
}
@@ -12,8 +12,8 @@ class OpenUserBatch {
def task = new OpenUserBatch();
// task.run();
// task.test();
// task.addShenZhenFjtUser();
task.addFujianFjtUser()
task.addShenZhenFjtUser();
// task.addFujianFjtUser()
}
def test(){
@@ -46,20 +46,20 @@ class OpenUserBatch {
}
def addShenZhenFjtUser(){
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
def i = 0;
for(def data : datas){
i++;
def id = 252608 + i;
def id = 252893 + i;
BigDecimal bd = new BigDecimal(data[3].toString());
String mobile = bd.toPlainString();
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
onLineSqlInstance.execute(sql);
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
onLineSqlInstance.execute(sql2)
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
onLineSqlInstance.execute(sql3)
}
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,102 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.google.gson.JsonObject
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.jsoup.Jsoup
class BaiduBaikeTown {
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
static main(args) {
def task = new BaiduBaikeTown()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
if(!row['E'] && row['A']=="郑州经济技术开发区"){
def q = new BasicDBObject()
q.put("desc",new BasicDBObject('$ne', 'null'))
q.put("name",row['A'])
def d = dzb_COLL.findOne(q)
def townMap = parse(d.desc, d.name)
row['D'] = townMap.town_str1
row['F'] = townMap.town_str2
// def ret = []
// ret << ["名称", "基本信息", "目录", "全文"]
// for(def d : dbcursor){
// def townMap = parse(d.desc, d.name)
// if(townMap.town){
// println d.name+" "+townMap.town
// }
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
// }
}
})
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
}
def parse(desc, name){
def map = [:]
if(desc){
def doc = Jsoup.parse(desc)
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
def s = doc.toString()
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
def jsonObj = JSONObject.fromObject(dataStr)
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
def towns = []
if(jsonObj && jsonObj.catalog){
def catalogs = jsonObj.catalog
def index = null
for(def log : catalogs){
def curIndex = log.index
if(log.title=="行政区划"){
index = log.index
}
if(index && curIndex.toString().startsWith(index+"-")){
towns << log.title
}
}
}
map.town_str1 = str
if(town){
map.town = town
}else{
map.town = ""
}
if(towns){
map.towns = towns.join(",")
}else{
map.towns = ""
}
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
map.town_str2 = str2
return map
}
map.town_str1 = ""
map.town_str2 = ""
map.town = ""
map.towns = ""
return map
}
}
@@ -7,6 +7,7 @@ import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.AddressUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
@@ -17,17 +18,23 @@ import net.sf.json.JSONObject
import org.apache.poi.ss.util.RegionUtil
import org.bson.types.ObjectId
import java.util.regex.Pattern
class DevProcess {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEV_MAPPING_CONN2 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping_2", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
static main(args) {
def task = new DevProcess();
// task.run()
@@ -41,6 +48,7 @@ class DevProcess {
// task.updateCityAndDist()
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
// task.addDevAddres2()//添加相差2公里范围内的地址
// task.addDevAddres3()
// task.matchDevAndAddAddress()
// task.stat()
@@ -48,7 +56,7 @@ class DevProcess {
// task.createDevMapping()
// task.deleteAddr()
// task.deleteAddr2()
// task.synchDevTo43()
// task.synchDevTo100()
// task.addNewDev2()
// task.addAlias()
@@ -56,6 +64,17 @@ class DevProcess {
// task.matchDevTownship()//添加乡镇街道
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
// task.matchNewCrawlDev()
// println DateUtils.add("2024-7-24", 7);
// task.addKfc()
// task.devToPark() //开发区下辖园区
// task.checkTown()
// task.repeatSave()
}
@@ -334,15 +353,27 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
if(!d['Q'] && d['L']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
if(!d['I'] && d['J']){
def q = new BasicDBObject()
q.put("name", d['L'])
q.put("city",d['G'])
// q.put("name", d['J'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['J']]))
def name = d['J'].replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['Q'] = obj._id.toString()
d['R'] = obj.name
d['I'] = "库中有开发区-相似2"
d['K'] = obj.name
// d['C'] = obj._id.toString()
// d['D'] = obj.name
// if(obj.name == d['A']){
// d['E'] = "相同"
// }else{
// d['E'] = "包含"
// }
// d['E'] = "别名"
// d['F'] = obj.name
}
}
},0);
@@ -509,6 +540,38 @@ class DevProcess {
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def addDevAddres3(){
def index = 0
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
if(d['A']){
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
d['C'] = obj._id.toString()
def q2 = new BasicDBObject()
q2.put("name", d['A'])
q2.put("address", d['B'])
q2.put("type","地址")
def sameObj = DEV_MAPPING_CONN.findOne(q2)
if(!sameObj){
index++
def udata = new BasicDBObject()
udata.put("province", obj.province)
udata.put("city", obj.city)
udata.put("district", obj.district)
udata.put("name",obj.name)
udata.put("type", "地址")
udata.put("address", d['B'])
DEV_MAPPING_CONN.save(udata)
}
}
}
},0)
println index
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
}
def stat(){
def dbcursor = development_zone_CONN.find(new BasicDBObject())
def total = 0
@@ -633,13 +696,13 @@ class DevProcess {
});
}
def synchDevTo43(){
def dbcursor = development_zone_CONN.find()
def synchDevTo100(){
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
def rets = []
for(def r : dbcursor){
rets << r
}
DEVELOPMENT_ZONE_43_COLL.insert(rets)
development_zone_CONN.insert(rets)
}
@@ -777,4 +840,151 @@ class DevProcess {
})
}
def matchNewCrawlDev(){
def query = new BasicDBObject()
query.put("match_id", new BasicDBObject('$exists', false))
def list = kfq_COLL.find(query).toList()
for(def d : list){
def q = new BasicDBObject()
// q.put("name", d.name)
q.put('alias', new BasicDBObject('$in',[d.name]))
def obj = development_zone_CONN.findOne(q)
if(obj){
def udata = new BasicDBObject()
udata.put("match_id", obj._id.toString())
udata.put("match_name", obj.name)
kfq_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
}
}
}
def addKfc(){
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\投促中国开发区(1).xlsx",{d->
if(d['I']){
if(d['I'] == "添加"){
def udata = new BasicDBObject()
udata.put("create_time" , new Date())
udata.put("industry",d['D'])
if(d['E'].contains("国家级")){
udata.put("level" ,"国家级")
}else if(d['E'].contains("省级")){
udata.put("level" ,"省级")
}
udata.put("name" ,d['F'])
udata.put("province" ,d['G'])
udata.put("type" ,d['H'])
udata.put("city" ,d['B'])
udata.put("district" ,d['J'])
development_zone_CONN.save(udata)
}else{
def q = new BasicDBObject()
q.put("name", d['I'])
def obj = development_zone_CONN.findOne(q)
if(obj){
def alias = []
if(obj.alias){
alias = obj.alias
}
if(!alias.contains(d['F'])){
alias << d['F']
}
if(alias && alias.size()!=obj.alias.size()){
def updateData = new BasicDBObject()
updateData.put("alias", alias)
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',updateData))
}
}
}
}
})
}
def devToPark(){
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1)
def rets = []
rets << ["开发区名称", "下辖街道", "城市", "区域"]
for(def d : datas){
if(d['E']){
def arrs = d['E'].split("、")
def q = new BasicDBObject().append("name", d['A'])
def obj = development_zone_CONN.findOne(q)
if(obj){
for(def park : arrs){
rets << [d['A'], park, obj.city, obj.district]
}
}
}
}
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道")
}
def checkTown(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d->
if(!d['E']){
if(d['A']){
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
// if(jsonObject1) {
// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
// def township = jsonObject1.get("township")
// def district = jsonObject1.get("district")
// d['E'] = "接口"
// if(district && district!='null'){
// d['F'] = district
// }
// if(township && township!='null'){
// d['G'] = township
// }
//
// d['H'] = jsonObject1.get("townCode")
// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
// d['I'] = jsonObject1.get("provinceCode")
// }
// }
// }
def q = new BasicDBObject()
q.put("city", d['C'])
// q.put("district", d['D'])
// q.put("mzb_district_name", d['D'])
q.put("type", "乡镇街道")
q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE)))
// q.put("street", d['B'])
// q.put('alias', new BasicDBObject('$in',[d['B']]))
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
d['E'] = "可入库-忽略区域模糊2"
d['F'] = obj.district
d['G'] = obj.street
}
}
}
},0)
}
def repeatSave(){
def dbcursor = DEV_MAPPING_CONN.find(new BasicDBObject())
def list = []
for(def map : dbcursor){
def columns = ["_id","province","city","district","name","type","address","dev_district", "township","park"]
def udata = new BasicDBObject()
map.each{k,v->
if(columns.contains(k)){
if(v instanceof String){
if(k=="_id"){
udata.put("_id", new ObjectId(v))
}else{
udata.put(k, v)
}
}
}
}
list << udata
}
DEV_MAPPING_CONN2.insert(list)
}
}
@@ -0,0 +1,281 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.AMQP
import net.sf.json.JSONObject
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
class DevUpdateTestBatch {
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
static main(args) {
def task = new DevUpdateTestBatch()
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
// task.addDev(path)
// task.addDevAddress(path)
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
}
def addDev(path){
ExcelUtils.modify(path,{ d->
if(!d['I'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(!obj){
def o = new BasicDBObject()
o.put("province", d['B'])
o.put("city", d['C'])
o.put("district", d['D'])
o.put("name", d['A'])
o.put("level", d['E'])
o.put("type", d['F'])
if(d['G']){
o.put("township", d['G'])
}
if(d['H']){
def arr = d['H'].split(",")
if(arr){
o.put("alias", arr)
}
}
def id = addMongoDB(o)
o.remove("_id")
o.id = id.toString()
o.remove("create_time")
addEs(o, "192.168.0.89", "development_zone")
addEs(o, "192.168.0.52", "development_zone")
d['I'] = "已完成"
}else{
println d['A'] +" 已存在"
}
}
})
}
def addMongoDB(o){
o.put("create_time", new Date())
development_zone_100_CONN.save(o)
DEVELOPMENT_ZONE_43_COLL.save(o)
return o._id
}
def addEs(map, host, indexName){
BulkRequest request = new BulkRequest();
int processedCount = 0;
def client = null
if(host && host == "192.168.0.89"){
client = ElasticUtils89.getClient(host)
}else if(host && host == "192.168.0.52"){
client = ElasticUtils.getClient(host)
}
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
if (request.numberOfActions() > 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
println "Processed " + processedCount + " documents"
}
}
}
def addDevAddress(path){
ExcelUtils.modify(path,{ d->
if(!d['D'] && d['A']){
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
if(obj){
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
if(d['B'] == "乡镇街道"){
q.put("township", d['C'])
}else if(d['B'] == "地址"){
q.put("address", d['C'])
}else if(d['B'] == "园区"){
q.put("park", d['C'])
}
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
if(!mappingObj){
def o = new BasicDBObject()
o.put("province", obj.province)
o.put("city", obj.city)
o.put("district", obj.district)
o.put("name", obj.name)
o.put("type", d['B'])
if(d['B'] == "乡镇街道"){
o.put("township", d['C'])
}else if(d['B'] == "地址"){
o.put("address", d['C'])
}else if(d['B'] == "园区"){
o.put("park", d['C'])
}
def date = new Date()
o.put("c_date", date)
o.put("version", date.getTime())
DEV_MAPPING_100_CONN.save(o)
// o.id = o._id.toString()
// o.remove("_id")
// o.remove("c_date")
// addEs(o, "192.168.0.89", "dev_mapping")
// try{
// addEs(o, "192.168.0.52", "dev_mapping")
// }catch(Exception e){
// println "52 es error: "+e.printStackTrace()
// }
d['D'] = "已完成"
}else{
d['D'] = "重复"
println d['A'] +" "+ d['C']+" 重复"
}
}else{
d['D'] = "不存在"
println d['A'] +" 不存在"
}
}
},1)
}
def updateHasGov(province, city, district, street, hasGov){
def q = new BasicDBObject()
q.put("province", province)
q.put("city", city)
q.put("district", district)
q.put("street", street)
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
//修改43
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
//修改89 administrative_mapping
update89EsAdministrativeMapping(province, city, district, street, hasGov)
//修改89 administrative_mapping
update52EsAdministrativeMapping(province, city, district, street, hasGov)
}else{
println province+" "+city+" "+district+" "+street+" 没找到"
}
//修改100 community
//修改89 community
//修改52 community
}
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
def header = [:]
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
def body = """
{
"script": {
"source": "ctx._source.has_gov='${hasGov}'",
"lang": "painless"
},
"query": {
"bool": {
"must": [
{
"match": {
"province_name": "${province}"
}
},
{
"match": {
"city_name":"${city}"
}
},
{
"match": {
"area_name":"${district}"
}
},
{
"match": {
"street_name":"${street}"
}
}
]
}
}
}
"""
HttpRequest request = new HttpRequest()
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
// def response = JSONObject.fromObject(res)
println(res)
}
}
@@ -10,7 +10,8 @@ class DistrictAliasProcess {
static main(args) {
def task = new DistrictAliasProcess();
task.run()
// task.run()
task.fenQiProcess()
}
def run(){
@@ -42,4 +43,36 @@ class DistrictAliasProcess {
}
})
}
def fenQiProcess(){
def q = new BasicDBObject()
q.put("cat","district")
q.put("status", 0)
q.put("master_district_name", 'master')
def dbcursor = DbUtils.seaweed.find(q)
def rets = []
for(def d : dbcursor){
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
}
for(def r : rets){
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
def lastObj = null
for(def j : rets){
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
lastObj = j;
break
}
}
if(lastObj){
println r
def udata = new BasicDBObject()
udata.put("master_district_name","")
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
}
}
}
}
@@ -28,23 +28,23 @@ class GXDdistrictMatch {
static main(args) {
def task = new GXDdistrictMatch();
// task.run()
// task.addOldMatch()
task.addOldMatch()
// task.runDb()
// task.updateDistStatus()//恢复name_cat不正常数据
// task.saveToDB() //保存新增小区
// task.restoreDist()//恢复小区status=0
task.runMatch()
// task.runMatch()
}
def addOldMatch(){
//老匹配
def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
ExcelUtils.multiThreadingModify(fileName, { d ->
println d['C']
if(!d['AJ']){
def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C'])
if(!d['AJ'] && d['I']){
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['AJ'] = jsonObject.get("id")
d['AK'] = jsonObject.get("city")
@@ -0,0 +1,347 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.regex.Pattern
/**
* 名录集
*/
class MinglujiData {
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL)
static main(args) {
def task = new MinglujiData()
task.run()
}
def run(){
def prov = "浙江省"
def city = "嘉兴"
// exportDatas(city)
// matchDev(prov, city)
matchCommunity(city)
// tempMatchDev(prov)
// saveMljData()
}
def exportDatas(city){
def datas = findMinglujiDatas(city)
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
}
def matchDev(prov, city){
def devs = loadAllDevs(prov)
def i = 1;
while(i<20) {
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName, { d ->
if (d['D']) {
def street = d['D']
def splitStreet = ""
if (street) {
if (street.contains(d['C'])) {
def start = street.toString().lastIndexOf(d['C']) + d['C'].size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
} else {
splitStreet = street
}
}
d['K'] = splitStreet
}
if (d['I']) {
def splitCommunity = ""
if (d['C'] && d['I'].toString().contains(d['C'])) {
splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size())
} else {
splitCommunity = d['I']
}
d['L'] = splitCommunity
}
if (d['E'] && !d['N']) {
def obj = findDevByName(d['E'])
if (!obj) {
obj = findDevByAlias(d['E'])
if (!obj) {
obj = findDebBySimlar(devs, d['E'], d['B'])
if (!obj) {
obj = findDevByPark(d['A'], d['B'], d['E'])
if (!obj) {
} else {
d['N'] = "园区"
}
} else {
d['N'] = "相似"
}
} else {
d['N'] = "别名"
}
} else {
d['N'] = "相同"
}
if (obj) {
d['M'] = obj.name
d['O'] = obj.city
d['P'] = obj.district
}
}
})
}
i++
println fileName +" 完成"
}
}
def tempMatchDev(prov){
def devs = loadAllDevs(prov)
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d->
if(d['U'] && !d['W'] ){
def obj = findDevByName(d['U'])
if(!obj){
obj = findDevByAlias(d['U'])
if(!obj){
obj = findDebBySimlar(devs, d['U'], d['B'])
if(!obj){
obj = findDevByPark(d['A'], d['B'], d['U'])
if (!obj) {
}else{
d['X'] = "园区"
}
}else{
d['X'] = "相似"
}
}else{
d['X'] = "别名"
}
}else{
d['X'] = "相同"
}
if(obj){
d['W'] = obj.name
d['Y'] = obj.city
d['Z'] = obj.district
}
}
})
}
def matchCommunity(city){
def i = 1;
while(i<20){
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
}
File file=new File(fileName);
if(file.exists()){
ExcelUtils.multiThreadingModify(fileName,{d->
if(d['A'] && d['B'] && d['L'] && !d['Q']){
def list = findCommunityByName(d['A'], d['B'], d['L'])
if(!list){
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
if(!list){
}else{
d['T'] = "包含"
}
}else{
d['T'] = "相同"
}
if(list){
if(list.size()==1){
def obj = list[0]
d['Q'] = obj.name
d['R'] = obj.street_name
d['U'] = obj.area_name
d['V'] = obj.city_name
}else if(list.size()>=2){
for(def l : list){
if(d['C']==l.area_name){
d['Q'] = l.name
d['R'] = l.street_name
d['U'] = l.area_name
d['V'] = l.city_name
break;
}
}
d['S'] = list.size()
}
}
}
}, 0)
println fileName +" 远行完成"
}
i++
}
}
def saveMljData(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d->
def obj = new BasicDBObject()
obj.put("province", d['A'])
obj.put("city", d['B'])
obj.put("district", d['C'])
obj.put("name", d['D'])
obj.put("address", d['E'])
obj.put("dev_name", d['F'])
obj.put("committee", d['G'])
obj.put("street", d['H'])
company_100_CONN.save(obj)
})
}
def loadAllDevs(prov){
def q = new BasicDBObject()
q.put("province", prov)
return development_zone_CONN.find(q).toList()
}
def findDebBySimlar(def devs, def name, def city){
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
def last = null
for(def d : devs){
def alias1 = null
if(!d.alias){
alias1 = []
}else{
alias1 = d.alias
}
if(!alias1.contains(d.name)){
alias1 << d.name
}
if(alias1){
try {
for(def n : alias1){
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
last = d
break
}
}
}catch(exception){
println alias1
}
}
if(last){
break
}
}
return last
}
def findDevByName(name){
def q = new BasicDBObject()
q.put("name", name)
return development_zone_CONN.findOne(q)
}
def findDevByAlias(name){
def q = new BasicDBObject()
q.put('alias', new BasicDBObject('$in',[name]))
return development_zone_CONN.findOne(q)
}
def findDevBySimlar(name){
def q = new BasicDBObject()
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
return development_zone_CONN.findOne(q)
}
def findDevByPark(prov, city, name){
def q = new BasicDBObject()
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
q.put("type", "园区")
return dev_mapping_CONN.findOne(q)
}
def findCommunityByName(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", name)
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findCommunityBySimlar(province, city, name){
def q = new BasicDBObject()
q.put("province_name", province+"省")
q.put("city_name", city+"市")
q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE)))
def dbcursor = community_100_CONN.find(q)
def list = []
for(def d : dbcursor){
list << d
}
return list
}
def findMinglujiDatas(city){
def keys = new BasicDBObject()
keys.put("c_address",1)
keys.put("c_city",1)
keys.put("c_committee",1)
keys.put("c_community",1)
keys.put("c_company",1)
keys.put("c_district",1)
keys.put("c_street",1)
keys.put("kind",1)
keys.put("name",1)
keys.put("province",1)
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
def rets = []
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
for(def d : datas){
def street = d.c_street
def splitStreet = ""
if(street){
if(street.contains(d.c_district)){
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
def end = street.toString().size()
splitStreet = street.toString().substring(start, end)
}
}
def splitCommunity = ""
if(d.c_committee){
if(d.c_committee.contains(d.c_district)){
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
}
}
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
}
return rets
}
}
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicLong
class PoiLocation {
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
@@ -35,21 +35,52 @@ class PoiLocation {
static main(args) {
def task = new PoiLocation();
task.run();
// task.run();
task.regionAndStreetProcess()
}
def run(){
def citys = ["芜湖市","重庆市","杭州市","泉州市","东莞市","福州市","厦门市","佛山市","天津市","武汉市","宁波市","郑州市","长沙市","青岛市","上海市","西安市","温州市","无锡市","南京市","湖州市","合肥市","临沂市","石家庄市","昆明市","金华市","保定市","成都市","济南市","北京市","沈阳市","潍坊市","广州市","惠州市","嘉兴市","赣州市","苏州市","南通市","南宁市","台州市","南阳市","徐州市","长春市","深圳市","哈尔滨市","漳州市","中山市","常州市","烟台市","大连市","邯郸市","济宁市","盐城市","绍兴市","唐山市","洛阳市","阜阳市","菏泽市","贵阳市","沧州市","遵义市","莆田市","南昌市","商丘市","周口市","邢台市","江门市","衡阳市","邵阳市","信阳市","廊坊市","扬州市","泰州市","新乡市","太原市","德州市","淄博市","宁德市","绵阳市","汕头市","聊城市","淮安市","驻马店市","黄冈市","宿迁市","三明市","六安市","龙岩市","桂林市","岳阳市","南平市","揭阳市","上饶市","九江市","南充市","连云港市","襄阳市","泰安市","常德市","郴州市","安阳市","安庆市","湛江市","永州市","茂名市","曲靖市","宜宾市","肇庆市","宿州市","兰州市","运城市","镇江市","滁州市","威海市","梅州市","宜春市","清远市","亳州市","怀化市","荆州市","平顶山市","开封市","玉林市","咸阳市","宜昌市","株洲市","乌鲁木齐市","滨州市","渭南市","临汾市","榆林市","银川市","衡水市","毕节市","枣庄市","吉安市","张家口市","柳州市","许昌市","泸州市","呼和浩特市","达州市","德阳市","赤峰市","珠海市","孝感市","益阳市","娄底市","乐山市","恩施土家族苗族自治州","焦作市","湘潭市","海口市","红河哈尼族彝族自治州","黔南布依族苗族自治州","河源市","汉中市","日照市","大理白族自治州","濮阳市","黔东南苗族侗族自治州","秦皇岛市","晋中市","宝鸡市","吉林市","包头市","蚌埠市","东营市","宣城市","韶关市","衢州市","十堰市","承德市","丽水市","眉山市","鞍山市","昭通市","凉山彝族自治州","长治市","抚州市","鄂尔多斯市","吕梁市","百色市","齐齐哈尔市","贵港市","潮州市","黔西南布依族苗族自治州","广安市","淮南市","河池市","大同市","铜仁市","阳江市","咸宁市","营口市","绥化市","马鞍山市","安康市","荆门市","忻州市","广元市","内江市","遂宁市","西宁市","大庆市","汕尾市","文山壮族苗族自治州","锦州市","延安市","巴中市","呼伦贝尔市","楚雄彝族自治州","黄石市","玉溪市","六盘水市","通辽市","钦州市","云浮市","梧州市","漯河市","晋城市","庆阳市","安顺市","丹东市","自贡市","朝阳市","葫芦岛市","天水市","三门峡市","湘西土家族苗族自治州","北海市","陇南市","普洱市","延边朝鲜族自治州","保山市","定西市","随州市","喀什地区","黄山市","佳木斯市","巴彦淖尔市","牡丹江市","舟山市","资阳市","商洛市","松原市","锡林郭勒盟","伊犁哈萨克自治州","萍乡市","铁岭市","乌兰察布市","丽江市","景德镇市","张家界市","抚顺市","通化市","淮北市","来宾市","临沧市","阿克苏地区","贺州市","盘锦市","昌吉回族自治州","池州市","雅安市","三亚市","平凉市","四平市","酒泉市","西双版纳傣族自治州","鹤壁市","辽阳市","白城市","吴忠市","崇左市","阜新市","朔州市","白银市","拉萨市","巴音郭楞蒙古自治州","张掖市","铜陵市","武威市","临夏回族自治州","德宏傣族景颇族自治州","新余市","甘孜藏族自治州","兴安盟","本溪市","鄂州市","黑河市","阿坝藏族羌族自治州","阳泉市","防城港市","鸡西市","仙桃市","鹰潭市","固原市","白山市","中卫市","双鸭山市","攀枝花市","石嘴山市","和田地区","海东市","塔城地区","日喀则市","济源市","伊春市","天门市","辽源市","潜江市","那曲市","铜川市","海西蒙古族藏族自治州","鹤岗市","昌都市","阿拉善盟","乌海市","儋州市","哈密市","克拉玛依市","七台河市","金昌市","甘南藏族自治州","琼海市","迪庆藏族自治州","石河子市","阿勒泰地区","文昌市","玉树藏族自治州","嘉峪关市","万宁市","怒江傈僳族自治州","林芝市","海南藏族自治州","吐鲁番市","澄迈县","陵水黎族自治县","博尔塔拉蒙古自治州","山南市","阿里地区","大兴安岭地区","海北藏族自治州","东方市","乐东黎族自治县","克孜勒苏柯尔克孜自治州","果洛藏族自治州","临高县","阿拉尔市","黄南藏族自治州","神农架林区","屯昌县","定安县","昌江黎族自治县","五家渠市","保亭黎族苗族自治县","琼中黎族苗族自治县","五指山市","白沙黎族自治县","北屯市","图木舒克市","可克达拉市","双河市","铁门关市","毕节地区","昆玉市","铜仁地区","海东地区","三沙市","吐鲁番地区","哈密地区","日喀则地区","林芝地区","昌都地区","山南地区","那曲地区"]
def citys = [""]
citys = citys.reverse()
for(def city : citys){
findPois(city)
}
}
def regionAndStreetProcess(){
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
for(def city : citys){
def datas = findAllPoisByCity(city)
process(datas)
}
}
def findAllPoisByCity(city){
def q = new BasicDBObject()
q.put("city", city)
def keys = new BasicDBObject()
keys.put("_id", 1)
keys.put("province", 1)
keys.put("city", 1)
keys.put("district", 1)
keys.put("township", 1)
keys.put("location", 1)
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findPois(city){
def q = new BasicDBObject()
q.put("city", city)
q.put("run_status", new BasicDBObject('$ne', 1))
// q.put("run_status", new BasicDBObject('$ne', 1))
def keys = new BasicDBObject()
keys.put("_id", 1)
@@ -87,6 +118,32 @@ class PoiLocation {
return rets;
}
def process(datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
dataRun(d)
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return rets
}
def checkData(def index, def datas){
def totalCount = new AtomicLong()
def rets = new CopyOnWriteArrayList()
@@ -190,6 +247,33 @@ class PoiLocation {
return map;
}
def dataRun(d){
def regions = getFullRegion(d)
if(d.location) {
//判断区域否在围栏里
def regionFlag = null
if (d.district) {
for (def rs : regions) {
if (rs.shape && rs.shape.coordinates) {
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
}
}
}
if (regionFlag) {//在围栏里
//判断township是否在统计局中
// if(d.township){
// town = getSimilarTown(regions, d.city, d.township)
// }
}else{//不在转栏里,重新搜索
println(d)
}
}
}
def getFullRegion(d){
def regions = regionMap.get(d.city)
def names = []
@@ -0,0 +1,159 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@BatchOwner(Watcher.GaoJuan)
class ThardParthAnalyBatch extends BaseBatch{
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new ThardParthAnalyBatch(), args)
}
// static main(args) {
// def task = new ThardParthAnalyBatch()
// task.run()
// }
@Override
void process(Object args) {
run()
}
def run(){
def fjwTotalCount = CommonDatautils.findAllCount("district")
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
println "统计月份:"+sDate+" 查询更新时间:"+mdate
def crawlTotalCount = findAllEsDataCount(mdate)
println "抓取第三方小区总量:"+crawlTotalCount
def synchTotalCount = findAllMongoDatasCount(sDate)
println "同步到第三方库小区总量: "+synchTotalCount
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
def synchDistCount = findAllMongoDistrictCount(sDate)
println "同步到第三方库有价格小区量: "+synchDistCount
def matchTotalCount = findMatchSuccessCount(sDate)
println "第三方匹配成功总量:"+matchTotalCount
def matchDistTotalCount = findMatchSuccessDistCount(sDate)
println "第三方匹配成功小区量:"+matchDistTotalCount
def precent = matchDistTotalCount*100.0/fjwTotalCount
precent = DataHandleUtils.getDoubleRound(2, precent)
println "匹配成功占比:"+precent+"%"
def noMatchTotalCount = findNoMatchCount(sDate)
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount);
}
def findAllEsDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllEsDistDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllMongoDatasCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
return thirdPartyColl.count(q)
}
def findAllMongoDistrictCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
q.put("avg_price", new BasicDBObject('$gt', 0))
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("region",1)
keys.put("name",1)
def dbcursor = thirdPartyColl.find(q,keys)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
}
return set.size()
}
def findAllMongoDatasHavePricesCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
return thirdPartyColl.count(q)
}
def findMatchSuccessCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
return thirdPartyColl.count(q)
}
def findMatchSuccessDistCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
def dbcursor = thirdPartyColl.find(q)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
}
return set.size()
}
def findNoMatchCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
return thirdPartyColl.count(q)
}
def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
def udata = new BasicDBObject()
udata.put("fjw_dist_total", fjwTotalCount)
udata.put("crawl_total", crawlTotalCount)
udata.put("synch_total", synchTotalCount)
udata.put("synch_have_price_total", synchHavePriceTotalCount)
udata.put("synch_have_price_district_total", synchDistCount)
udata.put("match_total", matchTotalCount)
udata.put("match_dist_total", matchDistTotalCount)
udata.put("match_dist_rate", precent)
udata.put("no_match_total", noMatchTotalCount)
udata.put("c_date", new Date())
if(obj){
udata.put("m_date", new Date())
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
}else{
udata.put("s_date", sDate)
THIRD_PARTY_STAT.save(udata)
}
}
}
@@ -2,8 +2,10 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.ProvinceData
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -12,6 +14,8 @@ import com.mongodb.client.AggregateIterable
import org.bson.Document
import org.bson.types.ObjectId
import java.util.regex.Pattern
/**
* 乡镇街道中间表
* 包含市,区,乡镇街道别名,来源 统计局community表
@@ -26,6 +30,9 @@ class TownMiddleProcess {
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection ADMINISTRATIVE_MAPPING_43base_COLL = MongoUtils.getCollectionByName("fangjia_base", "administrative_mapping", "mongo.db.fangjia.com")
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
@@ -67,7 +74,15 @@ class TownMiddleProcess {
// checkTown()
//区域民政部名称更新到乡镇街道中
// updateTownshipByRegion()
// addAliasFromExcel()
//增加简称及简拼
// addSimpleName()
//根据民政部数据修改行政区划中间表
// mzbDistrictUpate()
// addCity()
}
@@ -422,14 +437,16 @@ class TownMiddleProcess {
for(def d : dbcursor){
datas << d
if (datas.size() >= 2000) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
println datas.size()
datas.clear();
}
}
if (datas.size() >= 0) {
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
datas.clear();
}
}
@@ -929,4 +946,222 @@ class TownMiddleProcess {
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
}
}
def addAliasFromExcel(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道合并或者变更20240627.xlsx",{ row->
def q = new BasicDBObject("type", "乡镇街道")
q.put("city", row['A'])
q.put("street", row['D'])
if(row['B']){
q.put("district", row['B'])
}
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(obj){
row['F'] = '是'
def alias = obj.alias
if(!alias){
alias = []
}
if(!alias.contains(row['C'])){
alias << row['C']
}
if(alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
def updateQuery = new BasicDBObject("_id", obj._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
})
}
def addSimpleName(){
def q = new BasicDBObject()
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
for(def d : dbcursor){
def prov = ProvinceData.getSimpleProvince(d.province)
if(prov){
def udata = new BasicDBObject()
udata.put("province_short", prov)
def updateQuery = new BasicDBObject("_id", d._id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
// def city = d.city.replace("市","").replace("区","").replace("县","")
// def udata = new BasicDBObject()
// udata.put("city_short", city)
// if(d.district && d.district.size()>=3){
// def region= d.district.replace("区","").replace("镇","").replace("乡","")
// udata.put("district_short", region)
// }
// def updateQuery = new BasicDBObject("_id", d._id)
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
// def city = RegionData.formatCity(d.city)
// if(city && d.mzb_district_name){
// def region = RegionData.formatRegion(city, d.mzb_district_name)
// if(region){
// def udata = new BasicDBObject()
// udata.put("city_short", city)
// udata.put("district_short", region)
//
// def updateQuery = new BasicDBObject("_id", d._id)
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
// }
// }
}
}
def mzbDistrictUpate(){
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
if(!row['I']){
def updateType = row['E']
def prov = row['B']
def newStreet = row['G']
if(["新设街道","新设镇","新增街道","新建街道","新建乡镇","新设乡","新增乡","新增建制"].contains(updateType)){
def dbo = findStreets(prov, newStreet, row['F'].toString().toDouble().toInteger().toString())
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['K'] = "存在"
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
}
}else if(["撤乡设镇","撤乡建镇","撤乡设乡","撤乡、镇设镇","撤销设街道","撤销建街道","撤销街道建镇","撤镇设镇、街道","撤销镇新设镇","乡镇合并","整建制划归","撤镇建新镇","区划变更","撤镇、街道设镇","撤街道建镇","撤销乡、镇新设镇","撤镇、街道设街道","撤乡、街道设街道","撤乡设街道","撤乡建街道","撤镇设街道","撤镇建街道","名称变更","撤街道设镇","撤乡、镇设街道","撤街道、镇、乡","并入其他建制",'撤镇设镇',"合并","合并建镇","更名","撤销合并","合并建制","合并镇","合并街道","名称变更","合并调整","合并建街道","撤镇设街道、镇","撤乡、镇设乡","撤镇乡设街道","区划调整","合并乡","撤镇设市","区划代码调整","区划代码变更","撤销乡、镇设街道"].contains(updateType)){
if(row['F']){
def dbo = null
def code = row['F'].toString()
if(!code.contains(" ")){
dbo = findStreets(prov, newStreet, code.toDouble().toInteger().toString())
}
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
def alias = dbo.alias
if(!dbo.alias){
alias = []
}
if(row['D']){
if(!alias.contains(row['D'])){
row['K'] = "存在(加别名)"
alias << row['D']
addAlias(dbo._id, alias)
}else{
row['K'] = "存在"
}
}else{
row['K'] = "存在"
}
}
}
}else if(updateType == "撤街道设街道"){
row['K'] = "人工处理"
}else if(updateType == "隶属关系变更"){
row['K'] = "人工处理"
}else if(["撤销镇","撤乡","撤销街道","撤销","撤街道设社区工作站","撤销建制"].contains(updateType)){
row['K'] = "人工处理"
}else if(updateType =='撤销乡'){
if(!row['F']){
row['K'] = "人工处理"
}
}
if(prov && newStreet && (!row['K'] || (row['K'] && !row['K'].contains("存在")) )){
def dbo = findStreets(prov, newStreet, null)
if(dbo){
row['I'] = dbo.city
row['J'] = dbo.district
row['K'] = "存在-编码不同"
row['L'] = dbo.tjj_street_code
row['M'] = dbo.pid
def alias = dbo.alias
if(!dbo.alias){
alias = []
}
if(row['D']){
if(!alias.contains(row['D'])){
row['K'] = "存在(加别名)"
alias << row['D']
// addAlias(dbo._id, alias)
}else{
row['K'] = "存在"
}
}else{
row['K'] = "存在"
}
}
}
}
})
}
def addCity(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
def citycode = null
if(row['F']){
citycode = row['F'].toString().substring(0, 4)
}else if(row['C']){
citycode = row['C'].toString().substring(0, 4)
}
if(citycode){
def q = new BasicDBObject()
q.put("mzb_city_code", citycode+"00")
q.put("type", "城市")
def dbo = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
if(dbo){
row['O'] = dbo.city
}
}
})
}
def findStreets(prov, newStreet, streetCode){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(prov+'.*$', Pattern.CASE_INSENSITIVE);
q.put("province", new BasicDBObject('$regex', provPattern))
q.put("street", newStreet)
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
def list = []
def dbo = null
for(def d : dbcursor){
list << d
}
if(list && list.size()==1){
dbo = list[0]
}else if(list && list.size()>1){
if(streetCode){
for(def d : list){
if(d.tjj_street_code && d.tjj_street_code.contains(streetCode)){
dbo = d
break;
}else if(streetCode.contains(d.pid)){
dbo = d
break;
}
}
}
}
return dbo
}
def addAlias(id, alias){
def udata = new BasicDBObject()
udata.put("alias", alias)
udata.put("operator","高娟-批量")
def updateQuery = new BasicDBObject("_id", id)
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -9,12 +10,14 @@ import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class TownshipStrandUpdate {
@@ -26,13 +29,26 @@ class TownshipStrandUpdate {
def polylineMap = null
def newAdvMappingMap = null
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
static main(args) {
def task = new TownshipStrandUpdate()
// task.process()
task.updateTown()
// task.updateTown()
// task.stat()
// task.updateDiffRegionTown()
// task.cleanTown()
// task.checkTownFromMongo()
// task.updateTownMapping()
// task.updateDevMapping()
println new Date().getTime()
// task.updateSpecCityTown()
}
@@ -43,21 +59,21 @@ class TownshipStrandUpdate {
polylineMap = CommonDatautils.findAllTownRanges()
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
// def citys = ["鉗�","벳덱","柯소鏤","괏땍","해뽈돎","읜렌","꾕鹵","蠟소왯","뷜彊","饉憩","넓돠"]//CommonDatautils.findAllCity()
def citys = CommonDatautils.findAllCity()
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
citys = citys.reverse()
for(def city : citys){
//
if(["中山","东莞"].contains(city)){
continue
}
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
// if(["中山","东莞"].contains(city)){
// continue
// }
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
File file=new File(fileName);
if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
// if(!file.exists()){
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
dists = checkData(dists)
def datas = createDatas(dists)
ExcelUtils.write(fileName, datas, "${city}")
}
// }
}
}
@@ -162,6 +178,10 @@ class TownshipStrandUpdate {
def running(d){
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
regions = []
regions << fullCity.city
}
def gaodeMap = null
def town = null
def mapTown = null
@@ -217,6 +237,10 @@ class TownshipStrandUpdate {
}
if( gaodeMap.district && gaodeMap.district!='[]'){
mapRegion = gaodeMap.district
}else{
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
mapRegion = mapCity
}
}
if(gaodeMap.township && gaodeMap.township!='[]'){
mapTownship = gaodeMap.township
@@ -331,7 +355,7 @@ class TownshipStrandUpdate {
def names = []
if(regions){
for(def r : regions){
if(r.mzb_district_name.contains(d.region)){
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
names << r.district
}
}
@@ -381,7 +405,7 @@ class TownshipStrandUpdate {
}
}
}
if(district || count>10){
if(township || district || count>10){
stop = true
}else{
count ++
@@ -397,4 +421,212 @@ class TownshipStrandUpdate {
map.format_addr = format_addr
return map;
}
def updateDiffRegionTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
ExcelUtils.multiThreadingModify(fileName,{d->
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
d['AA'] = lists.size()
//只处理库中只有一个的
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
def newRegion = RegionData.formatRegion(d['B'], d['M'])
if(newRegion){
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
def dbo = lists[0]
def arrs = d['X'].split(" ")
def udata = new BasicDBObject()
udata.put("region", newRegion)
udata.put("m_date", new Date())
udata.put("township", arrs[1])
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
d['AB'] = "已处理"
}
}
}
})
}
def cleanTown(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['AC']){
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
if(dbo && dbo.township){
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
}
}
})
}
def checkTownFromMongo(){
def citys = CommonDatautils.findAllCity()
fullCityMap = CommonDatautils.queryFullNameByCity()
regionMap = CommonDatautils.queryDistNameByCity()
commMap = CommonDatautils.queryCommunityByCity()
citys = citys.reverse()
def index = 0
for(def city : citys) {
index++
if (["中山", "东莞"].contains(city)) {
continue
}
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
for (def d : dists) {
if (d.township) {
def fullCity = fullCityMap.get(d.city)
def regions = getFullRegion(fullCity, d)
def town = getSimilarTown(regions, fullCity, d.township)
if (!town) {
town = findStreet(d.city, d.city, d.township)
if (!town) {
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", "")
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
println d.city + " " + d.region + " " + d.township + " " + d.name
}
}
}
}
println index + " " + city + " 完成"
}
}
def findStreet(city, district, street){
def q = new BasicDBObject()
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
q.put("city", new BasicDBObject('$regex', provPattern))
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
if(list){
if(list.size()==1){
return list[0]
}else{
println city+ " "+district+" "+street+" 有多个"
}
}
return null
}
def updateTownMapping(){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
ExcelUtils.modify(fileName,{d->
if(d['F']){
// def udata = new BasicDBObject()
// udata.put("mzb_district_name", d['F'])
//
// def updateQuery = new BasicDBObject()
// updateQuery.put("city", d['C'])
// updateQuery.put("district", d['D'])
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
def city = RegionData.formatCity(d['C'])
if(city){
def region = RegionData.formatRegion(city, d['D'])
if(region){
def updateQuery = new BasicDBObject()
updateQuery.put("category", "region")
updateQuery.put("city", city)
updateQuery.put("name", region)
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
if(list && list.size()==1){
def dbo = list[0]
def query = new BasicDBObject()
query.put("_id", dbo._id)
// def pattern = dbo.pattern
// if(pattern.contains(d['F'])){
// pattern.remove(d['F'])
// }
def udata = new BasicDBObject()
udata.put("org_name", d['F'])
// udata.put("pattern",pattern)
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
}else{
println city +" "+region+" "+list.size()
}
}
}
}
})
}
def updateDevMapping(){
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
ExcelUtils.modify(fileName,{d->
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("district", d['B'])
q.put("name",d['C'])
q.put("type", "乡镇街道")
q.put("township", d['D'])
def list = dev_MAPPING_COLL_100.find(q).toList()
if(list && list.size()==2){
def obj = list[0]
def removeQuery = new BasicDBObject()
removeQuery.put("_id", obj._id)
dev_MAPPING_COLL_100.remove(removeQuery)
}else{
println q
}
})
}
def updateSpecCityTown(){
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
// def citys = ["中山"]
for(def city : citys){
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
ExcelUtils.modify(fileName,{d->
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
if(list.size()>1){
println d['A'] +" "+d['C'] + " 有多个"
}else if(list.size()==1){
def dbo = list[0]
def town = null
// if(d['X']){
// town = d['X']
// }else
if(d['R']){
town = d['R']
}
if(!town && d['M']){
town = d['M']
}
def region = RegionData.formatRegion(d['A'], town)
if(region){
def q = new BasicDBObject()
q.put("region", region)
def udata = new BasicDBObject()
udata.put("m_date", new Date())
udata.put("township", town)
udata.put("region", region)
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
}else{
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
}
}
})
}
}
}
@@ -25,7 +25,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -33,7 +33,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
//235为房价网内部计算均价的库
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -53,31 +53,33 @@ class FjwBigdataCommunityBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def index = 0
def today = new Date().getDate()
def fangjiaDists = new HashSet()
println "starting 。。。。。。。。。"
for(def city : citys){
index ++
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
for(def d : dists){
fangjiaDists << d._id.toString()
}
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rets = addDeleteDist(dists, ccbMaps, rets)
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
rows.addAll(rets)
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
}
if(today > 10){
// if(today > 10){
delPgData()
saveToPgSql(rows)
}
// }
}
def addDeleteDist(dists, ccbMaps, rets){
def fangjiaDists = []
for(def d : dists){
fangjiaDists << d._id.toString()
}
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
def ccbDists = ccbMaps.keySet()
for(def ccbId : ccbDists){
@@ -40,7 +40,7 @@ class FjwBigdataDealBatch extends BaseBatch{
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -515,13 +515,12 @@ class FjwBigdataDealBatch extends BaseBatch{
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
@@ -67,12 +67,12 @@ class FjwBigdataForsaleBatch extends BaseBatch{
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
// 最大重试次数
private static final int MAX_TRY_COUNT = 5;
@@ -93,10 +93,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def endDate = curMonth+"19" //正式运行时curMonth要改
println "房源时间:"+startDate+" "+endDate
def houseIdsfilter = findAllHouseIdsByCity()
def houseIdsfilter = null;//findAllHouseIdsByCity()
for(def city : citys){
index++
houseIdsfilter = findAllHouseIdsByCity(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
@@ -491,9 +492,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
@@ -918,11 +919,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
return houseIds;
}
def findAllHouseIdsByCity(){
def findAllHouseIdsByCity(cityid){
// 创建布隆过滤器对象
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_id='${cityid}'",{it->
filter.put(it.gujia_sell_id)
});
@@ -34,7 +34,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
@@ -48,19 +48,24 @@ class FjwGuiDingPriceBatch extends BaseBatch{
def rows = []
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
def priceIdsfilter = FjwPgsqlCommUtils.findGuiDingIdsByCity()
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01")
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
if(prices){
def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
rows.addAll(rets)
println city + " " + rets.size()
}
}
println "【房价网内部】人工指导价总计:"+rows.size()
saveToPgSql(rows)
}
@@ -118,11 +123,15 @@ class FjwGuiDingPriceBatch extends BaseBatch{
return obj
}
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
def rets = []
if(prices){
for(def ps : prices){
def dist = distMap.get(ps.region+"##"+ps.name)
if(priceIdsfilter.mightContain(ps._id.toString())){
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
continue
}
if(dist){
def ret = []
def areaData = distCodeMap.get(dist.region)
@@ -1,7 +1,11 @@
package com.fangjia.script.gj.fjwPriceBaseData
import com.google.common.hash.BloomFilter
import com.google.common.hash.Funnels
import groovy.sql.Sql
import java.nio.charset.StandardCharsets
/**
* 房价网内部使用
*/
@@ -9,7 +13,7 @@ class FjwPgsqlCommUtils {
public static def database = "fangjia"
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified",
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
public static def findAllDistrictIdsByOracle(def cityid){
@@ -35,5 +39,14 @@ class FjwPgsqlCommUtils {
sqlInstance.executeUpdate(sql2)
}
public static def findGuiDingIdsByCity(){
// ´´½¨²¼Â¡¹ýÂËÆ÷ÔÏó
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
sqlInstance.eachRow("SELECT gujia_fsid FROM base.guided_price",{it->
filter.put(it.gujia_fsid)
});
return filter
}
}
@@ -35,11 +35,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
@@ -50,20 +47,23 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
def stop = false
def month = -1
while(!stop){
if(sDate<202004){
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
println weekend.toString() +" "+sDate
runDatas(citys)
}
month = month -1
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
if(sDate == 201912){
stop = true;
}
}
runDatas(citys)
//ͬ²½ÀúÊ·Êý¾Ý
// def stop = false
// def month = -1
// while(!stop){
// if(sDate>202404 && sDate<202407){
// def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
// println weekend.toString() +" "+sDate
// runDatas(citys)
// }
// month = month -1
// sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
// if(sDate == 201912){
// stop = true;
// }
// }
}
def runDatas(citys){
@@ -265,6 +265,10 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
if(third.get("5i5j_esf_price")){
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
}
if(third.get("fccs_esf_price")){
price_map.fccs_avg_price = third.get("fccs_esf_price")
}
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
allRows << dataMap
}
@@ -311,6 +315,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}else if(k=="fccs_avg_price"){
web = "·¿²ú³¬ÊÐ"
}
ret << sourceMap.get(web)
@@ -389,139 +395,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
return rets;
}
def createData3(cityid){
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def dataMap = [:]
dataMap.COMMID = r.COMMID
dataMap.COMMNAME = r.COMMNAME
dataMap.CITYID = r.CITYID
dataMap.CITYNAME = r.CITYNAME
dataMap.DISTID = r.DISTID
dataMap.DISTNAME = r.DISTNAME
dataMap.YEAR_MONTH = r.YEAR_MONTH
def price_map = [:]
dataMap.price_map = price_map
if(r.ZHUGE_AVG_PRICE){
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
}
if(r.ANJUKE_AVG_PRICE){
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
}
if(r.FANG_AVG_PRICE){
price_map.fang_avg_price = r.FANG_AVG_PRICE
}
if(r.LIANJIA_AVG_PRICE){
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
}
if(r.CITYHOUSE_AVG_PRICE){
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
}
if(r.MAITIAN_AVG_PRICE){
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
}
if(r.KE_AVG_PRICE){
price_map.ke_avg_price = r.KE_AVG_PRICE
}
if(r.TAOFANG_AVG_PRICE){
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
}
if(r.ZHONGYUAN_AVG_PRICE){
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
}
if(r.TBS_AVG_PRICE){
price_map.tbs_avg_price = r.TBS_AVG_PRICE
}
if(r.QFANG_AVG_PRICE){
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
}
if(r.FANGTAN_AVG_PRICE){
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
}
if(r.FANGDUODUO_AVG_PRICE){
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
}
if(r.WIWJ_AVG_PRICE){
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
}
dataMap.updatetime = r.UPDATETIME
allRows << dataMap
});
def results = []
for(def m : allRows){
def priceMap = m.price_map
priceMap.each{k,v->
def ret = []
ret << new ObjectId().toString()
ret << m.CITYID
ret << m.DISTID
ret << m.COMMID
ret << m.COMMNAME
def web = ""
if(k=="zhuge_avg_price"){
web = "诸葛找房"
}else if(k=="anjuke_avg_price"){
web = "安居客"
}else if(k=="fang_avg_price"){
web = "房天下"
}else if(k=="lianjia_avg_price"){
web = "链家"
}else if(k=="cityhouse_avg_price"){
web = "禧泰数据"
}else if(k=="maitian_avg_price"){
web = "麦田"
}else if(k=="ke_avg_price"){
web = "贝壳"
}else if(k=="taofang_avg_price"){
web = "淘房365"
}else if(k=="zhongyuan_avg_price"){
web = "中原"
}else if(k=="tbs_avg_price"){
web = "兔博士"
}else if(k=="qfang_avg_price"){
web = "Q房"
}else if(k=="fangtan_avg_price"){
web = "房探"
}else if(k=="fangduoduo_avg_price"){
web = "房多多"
}else if(k=="wiwj_avg_price"){
web = "我爱我家"
}
ret << sourceMap.get(web)
ret << web
ret << m.YEAR_MONTH
ret << v
ret << ""
ret << ""
ret << m.updatetime
results << ret
}
}
return results
}
def findThirdPartyPrice(def city, def statMap){
def q = new BasicDBObject()
@@ -592,6 +465,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
@@ -33,11 +33,11 @@ class AddPriceTest {
task.run();
}
def run(){
def filePath = "D:\\数据\\建行\\分期小区对比结果.xlsx"
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
// addGXDPrice(filePath)
// addCcbPrice(filePath)
addCcbPrice(filePath)
// addThirdPrice(filePath)
addFjwPrice(filePath)
// addFjwPrice(filePath)
// exportPkPrice()
// addRegion()
// addTradePrice(filePath)
@@ -167,14 +167,14 @@ class AddPriceTest {
}
def addCcbPrice(filepath){
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(filepath,{ d->
if(d['A']){
// def cityId = cityCodeDataMap.get(d['N'])
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
if(d['U']){
def cityId = cityCodeDataMap.get(d['V'])
def map = findAvgPrice(d['U'], cityId)
if(map){
// d['AK'] = map.avg_price
d['P'] = map.calc_type
d['F'] = map.avg_price
// d['P'] = map.calc_type
// d['AM'] = map.credit_value
}
}
@@ -183,7 +183,7 @@ class AddPriceTest {
def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
@@ -272,23 +272,23 @@ class AddPriceTest {
ExcelUtils.multiThreadingModify(path,{d->
// if(!d['L']){
// if(d['J'] instanceof String){
def dist = CommonDatautils.findDistrictById(d['J'])
def dist = CommonDatautils.findDistrictById(d['AJ'])
if(dist){
d['A'] = dist.city
d['B'] = dist.region
d['C'] = dist.name
d['D'] = dist.status
d['E'] = dist.name_cat
d['F'] = dist.estate_type2
d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
d['H'] = dist.master_district_name
// d['A'] = dist.city
// d['B'] = dist.region
d['E'] = dist.name
// d['D'] = dist.status
// d['E'] = dist.name_cat
// d['F'] = dist.estate_type2
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
// d['H'] = dist.master_district_name
// d['N'] = dist.region
// def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
// if(dbo){
// d['D'] = dbo.avg_price
//// d['X'] = dbo.cal_type
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
if(dbo){
d['G'] = dbo.avg_price
// d['X'] = dbo.cal_type
// d['E'] = dbo.avg_price_2b
// }
}
}
// }
// }
@@ -27,8 +27,8 @@ class QiliangTest {
// task.run()
def data = [
'address': '宁德市福安市中兴西路16号',
'type': '2' //1:对公 0:对私
'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号',
'type': '1' //1:对公 0:对私
]
def s = task.getProxyResult(data);
print(s);
@@ -0,0 +1,265 @@
package com.fangjia.script.gj.xiamen
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticUtils89 {
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
// connectionProperties.setProperty("user","developer");
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
connectionProperties.setProperty("user","elastic");
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
dataSource.setProperties(connectionProperties);
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
}
rs.getMetaData().getColumnCount()
while(rs.next()){
def row = [:]
for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
rs.close()
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
static RestHighLevelClient getClient89(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 180);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
* @return
*/
static createBulkProcessor(String host, RestHighLevelClient client = null) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
};
if(!client){
client = getClient(host);
}
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
final SearchHit hit = searchHits[k];
executorService.submit(new Runnable() {
@Override
void run() {
try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
e.printStackTrace()
}
}
})
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def results = []
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
SearchHit searchHit = searchHits[k]
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
if(results.size()%100000==0){
println "."+ new Date()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
client.close()
return results;
}
}
File diff suppressed because one or more lines are too long
@@ -129,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
def run(){
fullCityMap = CommonDatautils.queryFullNameCity()
mcaDistMap = CommonDatautils.queryDistNameCity()
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
// priceMap = findAllDistrictSellPrice()
def citys = CommonDatautils.findAllCity().reverse()
@@ -61,10 +61,10 @@ class DataStandardBatch extends BaseBatch{
private static def districtMap = [:];
private static def limit = 5000;
def static final SHORT_DATE = Utils.getLastMonth()
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static {
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
//查找出所有未进行匹配的数据
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
// qdata.put("city","福州")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
// def esSources = ["搜房":"搜房"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
static main(args) {
runBatch(new SynchDistrictPrice(), args);
@@ -32,7 +32,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
@@ -0,0 +1,69 @@
package com.fangjia.script.util
import javax.crypto.Cipher;
import javax.crypto.KeyGenerator;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.util.Base64;
public class DESUtil {
private static final String ALGORITHM = "DES";
// 生成密钥
public static SecretKey generateKey() throws Exception {
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
keyGenerator.init(56); // DES密钥长度为56位
return keyGenerator.generateKey();
}
// 将密钥转换为字符串
public static String keyToString(SecretKey secretKey) {
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
}
// 将字符串转换为密钥
public static SecretKey stringToKey(String keyStr) {
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
}
// DES加密
public static String encrypt(String data, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
byte[] encryptedData = cipher.doFinal(data.getBytes());
return Base64.getEncoder().encodeToString(encryptedData);
}
// DES解密
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
Cipher cipher = Cipher.getInstance(ALGORITHM);
cipher.init(Cipher.DECRYPT_MODE, secretKey);
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
byte[] decryptedData = cipher.doFinal(decodedData);
return new String(decryptedData);
}
public static void main(String[] args) {
try {
// 生成密钥
// SecretKey secretKey = generateKey();
// String keyStr = keyToString(secretKey);
// System.out.println("密钥: " + keyStr);
//
//// // 加密数据
// String data = "高娟";
// String encryptedData = encrypt(data, secretKey);
// System.out.println("加密后的数据: " + encryptedData);
// 解密数据
SecretKey newSecretKey = stringToKey("tnb12345");
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
System.out.println("解密后的数据: " + decryptedData);
} catch (Exception e) {
e.printStackTrace();
}
}
}
@@ -71,6 +71,10 @@ class MongoUtils {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
}
if ((domainUrl == "192.168.0.206") && (port == 27020)) {
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name)
}
def key = db + name + domainUrl + port
if (collectionMap.containsKey(key)) {
return collectionMap[key]
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
import java.util.zip.GZIPInputStream
class GaoDeApiUtils {
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //宝剑
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
/**
* 冪帠僅蛌華硊<br>
@@ -1038,7 +1038,7 @@ public class EvaluationServiceImpl implements EvaluationService {
SqmetreRent sqmetreRent = sqmetreRentService.findNearestSqmetreRent(district.getCity(), district.getRegion(), district.getName(), DateUtil.formatToYYYYMMDD(weekend), "district");
if(sqmetreRent!=null){
long totalPrice = evaluationRateService.evaluationRent(sqmetreRent.getAvgPrice(), evaluation);
long totalPrice = evaluationRateService.evaluationRent(sqmetreRent.getAvgPrice().intValue(), evaluation);
evaluation.setTotalPrice(((Number) (totalPrice)).intValue());
@@ -194,7 +194,7 @@ public class HouseReportServiceImpl implements HouseReportService{
SqmetreRent citySqmetreRent = sqmetreRentService.findNearestSqmetreRent(city, null, null, "city");
RegionStat cityRegionStat = regionStatService.findByCity(city);
if(null != citySqmetreRent && null != cityRegionStat){
Double rate = DataUtil.getDoubleRound(2, citySqmetreRent.getAvgPrice()*12*100/cityRegionStat.getAvgPrice());
Double rate = DataUtil.getDoubleRound(2, citySqmetreRent.getAvgPrice().intValue()*12*100/cityRegionStat.getAvgPrice());
return rate;
}
return null;
@@ -205,7 +205,7 @@ public class HouseReportServiceImpl implements HouseReportService{
SqmetreRent regionSqmetreRent = sqmetreRentService.findNearestSqmetreRent(city, region, null, "region");
RegionStat regionStat = regionStatService.findRegionStat(city, region);
if(null != regionSqmetreRent && null != regionStat){
Double rate = DataUtil.getDoubleRound(2, regionSqmetreRent.getAvgPrice()*12*100/regionStat.getAvgPrice());
Double rate = DataUtil.getDoubleRound(2, regionSqmetreRent.getAvgPrice().intValue()*12*100/regionStat.getAvgPrice());
return rate;
}
return null;
@@ -224,7 +224,7 @@ public class HouseReportServiceImpl implements HouseReportService{
distAvgPrice = statBo.getVillaDistrictStat().getAvgPrice();
}
if(distAvgPrice > 0){
Double rate = DataUtil.getDoubleRound(2, distSqmetreRent.getAvgPrice()*12*100/distAvgPrice);
Double rate = DataUtil.getDoubleRound(2, distSqmetreRent.getAvgPrice().intValue()*12*100/distAvgPrice);
return rate;
}
}
@@ -126,6 +126,13 @@ public class SqmetreRentServiceImpl implements SqmetreRentService{
if(StringUtils.isNotBlank(cat)){
params.add(new QueryParam("cat", cat));
}
if(null != startDate){
params.add(new QueryParam("weekend",QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
}
if(null != endDate){
params.add(new QueryParam("weekend",QueryParam.LESS_THAN_EQUAL_OPERATOR, endDate));
}
List<SortParam> sortParams = new ArrayList<SortParam>();
sortParams.add(new SortParam("weekend", true));
@@ -17,5 +17,6 @@ public interface VillageIdentifyService {
int count(String villageType,String city1,String address);
List<VillageIdentify> findAll(String villageType,String city1,String address,Pager pager);
List<VillageIdentify> findByCityAndAddress(String villageType,String city1,String address,Pager pager);
}
@@ -62,6 +62,23 @@ public class VillageIdentifyServiceImpl implements VillageIdentifyService {
return villageIdentifyDao.findAllByParams(queryParams, pager.getStartRow(), pager.getPageSize());
}
@Override
public List<VillageIdentify> findByCityAndAddress(String villageType, String city1, String address, Pager pager) {
List<QueryParam> queryParams = new ArrayList<QueryParam>();
if (StringUtils.isNotBlank(villageType)) {
queryParams.add(new QueryParam("villageType", villageType));
}else {
queryParams.add(new QueryParam("villageType", "¶Ô¹«"));
}
if (StringUtils.isNotBlank(city1)) {
queryParams.add(new QueryParam("cityParam", city1));
}
if (StringUtils.isNotBlank(address)) {
queryParams.add(new QueryParam("address", address));
}
return villageIdentifyDao.findAllByParams(queryParams, pager.getStartRow(), pager.getPageSize());
}
public void setVillageIdentifyDao(VillageIdentifyDao villageIdentifyDao) {
this.villageIdentifyDao = villageIdentifyDao;
}
@@ -1,5 +1,7 @@
package com.fangjia.service.open;
import java.util.Date;
public class StatAnalyBo {
private String key;
@@ -13,6 +15,17 @@ public class StatAnalyBo {
private Integer billingCount;
private Integer purpose;
private Double successRate;
// 新增剩余使用次数和到期时间
private Integer surplusCallCount;
private Date deadline;
// 新增url和name
private String url;
private String name;
// 添加日均使用量
private Double dailyUse;
public String getKey() {
return key;
}
@@ -85,4 +98,44 @@ public class StatAnalyBo {
public void setPurpose(Integer purpose) {
this.purpose = purpose;
}
public Integer getSurplusCallCount() {
return surplusCallCount;
}
public void setSurplusCallCount(Integer surplusCallCount) {
this.surplusCallCount = surplusCallCount;
}
public Date getDeadline() {
return deadline;
}
public void setDeadline(Date deadline) {
this.deadline = deadline;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Double getDailyUse() {
return dailyUse;
}
public void setDailyUse(Double dailyUse) {
this.dailyUse = dailyUse;
}
}
@@ -481,5 +481,12 @@
@media(max-device-width:375px){
.bg01{height: 1550px;}
.label_title01 {font-size: 1.2rem;}
.evaluationBox .evaluation_title {font-size: 1.2rem;}
.sample_icon05 {background-size: 350px 200px;}
}
@media(max-device-width:414px){
.sample_icon05 {background-size: 400px 200px;}
}