Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
+242
-15
@@ -1,11 +1,13 @@
|
||||
package com.fangjia.bkoff.controllers;
|
||||
|
||||
import com.alibaba.excel.EasyExcel;
|
||||
import com.alibaba.excel.ExcelWriter;
|
||||
import com.alibaba.excel.util.DateUtils;
|
||||
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||
import com.fangjia.bkoff.model.ApiUserModel;
|
||||
import com.fangjia.bkoff.model.JsonModel;
|
||||
import com.fangjia.core.util.BeanUtils;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
|
||||
import com.fangjia.dal.mongo.model.ApiInvokeStatistical;
|
||||
import com.fangjia.dal.mongo.model.ApiUserCfg;
|
||||
@@ -27,23 +29,33 @@ import com.fangjia.service.sql.UserService;
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.commons.lang.time.DateUtils;
|
||||
import org.apache.poi.hssf.usermodel.*;
|
||||
import org.apache.poi.ss.usermodel.Cell;
|
||||
import org.apache.poi.ss.usermodel.Row;
|
||||
import org.apache.poi.ss.usermodel.Sheet;
|
||||
import org.apache.poi.ss.usermodel.Workbook;
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.json.JSONArray;
|
||||
import org.sothis.core.util.CollectionUtils;
|
||||
import org.sothis.web.mvc.ActionContext;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
import scala.Int;
|
||||
|
||||
import javax.servlet.ServletOutputStream;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.math.BigDecimal;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.DecimalFormat;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.*;
|
||||
|
||||
|
||||
/**
|
||||
* api用户开通等管理controller
|
||||
*
|
||||
@@ -61,8 +73,6 @@ public class ApiUserController {
|
||||
private ApiInvokeStatisticalService apiInvokeStatisticalService;
|
||||
private ApiVisitCountService apiVisitCountService;
|
||||
private ApiCountCityService apiCountCityService;
|
||||
|
||||
|
||||
/**
|
||||
* 所有api用户的信息展示
|
||||
*
|
||||
@@ -303,14 +313,15 @@ public class ApiUserController {
|
||||
user.put("password", DigestUtils.md5Hex(model.getPassword()));
|
||||
//
|
||||
// // 更改用户密码
|
||||
this.userService.updateById(user,userId);
|
||||
this.userService.updateById(user, userId);
|
||||
// 刷新缓存
|
||||
this.apiUserService.clearApiUserAuthorityCache(userId);
|
||||
|
||||
model.setViewType("redirect");
|
||||
model.setViewParam("location", "/apiUser/edit?userId="+userId);
|
||||
model.setViewParam("location", "/apiUser/edit?userId=" + userId);
|
||||
return model;
|
||||
}
|
||||
|
||||
public Object editPwdAction(ApiUserModel model) {
|
||||
Long userId = model.getUserId();
|
||||
User user = this.userService.findById(userId);
|
||||
@@ -975,7 +986,21 @@ public class ApiUserController {
|
||||
} else {
|
||||
cities = new ArrayList<String>();
|
||||
}
|
||||
List<OpenUriAnaly> analys = openUriAnalyService.findOpenUriAnalyLogs(model.getUserId(), null, startDate, endDate);
|
||||
// 只查询正式账号
|
||||
List<QueryParam> queryParams = Arrays.asList(new QueryParam("isTest", 0));
|
||||
List<ApiUser> apiUsers = this.apiUserService.findApiUsersByParamsSortDate(queryParams);
|
||||
// 获取正式账号的ids
|
||||
List<Integer> ids = new ArrayList<Integer>();
|
||||
for (ApiUser apiUser : apiUsers) {
|
||||
ids.add((int) apiUser.getId());
|
||||
}
|
||||
List<OpenUriAnaly> analyAll = openUriAnalyService.findOpenUriAnalyLogs(model.getUserId(), null, startDate, endDate);
|
||||
List<OpenUriAnaly> analys = new ArrayList<OpenUriAnaly>();
|
||||
for (OpenUriAnaly analy : analyAll) {
|
||||
if (ids.contains(analy.getApiUserId())) {
|
||||
analys.add(analy);
|
||||
}
|
||||
}
|
||||
List<StatAnalyBo> bos = null;
|
||||
List<EachCityBo> eachCityBos = null;
|
||||
if (model.getWay() == null) {
|
||||
@@ -1016,9 +1041,7 @@ public class ApiUserController {
|
||||
}
|
||||
}
|
||||
}
|
||||
List<Long> userIds = new ArrayList<Long>();
|
||||
for (StatAnalyBo boTemp : bos) {
|
||||
userIds.add(Long.valueOf(boTemp.getUserId()));
|
||||
List<ApiVisitCount> apiVisitCounts = apiVisitCountService.findAllApiVisitCount(Long.valueOf(boTemp.getUserId()), null, startDate, endDate);
|
||||
int total = 0;
|
||||
for (ApiVisitCount apiVisitCount : apiVisitCounts) {
|
||||
@@ -1027,7 +1050,6 @@ public class ApiUserController {
|
||||
boTemp.setBillingCount(total);
|
||||
}
|
||||
|
||||
List<ApiUser> apiUsers = apiUserService.findApiUserByIds(userIds);
|
||||
Map<Long, ApiUser> idUserMap = new HashMap<Long, ApiUser>();
|
||||
for (ApiUser apiUser : apiUsers) {
|
||||
idUserMap.put(apiUser.getId(), apiUser);
|
||||
@@ -1051,8 +1073,199 @@ public class ApiUserController {
|
||||
return model;
|
||||
}
|
||||
|
||||
public Object exportAction(ApiUserModel model, HttpServletResponse response) throws IllegalAccessException {
|
||||
Date startDate = null;
|
||||
Date endDate = null;
|
||||
if (StringUtils.isBlank(model.getBegDate()) && StringUtils.isBlank(model.getEndDate())) {
|
||||
startDate = DateUtil.add(DateUtil.formatToYYYYMMDD(new Date()), -7);
|
||||
endDate = DateUtil.parseStringToDate(DateUtil.formatToYYYYMMDD(new Date()));
|
||||
} else {
|
||||
if (StringUtils.isNotBlank(model.getBegDate())) {
|
||||
startDate = DateUtil.parseStringToDate(model.getBegDate());
|
||||
}
|
||||
if (StringUtils.isNotBlank(model.getEndDate())) {
|
||||
endDate = DateUtil.parseStringToDate(model.getEndDate());
|
||||
}
|
||||
|
||||
public Object userVisitStatAction(ApiUserModel model) {
|
||||
}
|
||||
if (startDate != null) {
|
||||
model.setBegDate(DateUtil.formatToYYYYMMDD(startDate));
|
||||
}
|
||||
|
||||
if (endDate != null) {
|
||||
model.setEndDate(DateUtil.formatToYYYYMMDD(endDate));
|
||||
}
|
||||
|
||||
long diffInMillies = Math.abs(endDate.getTime() - startDate.getTime());
|
||||
|
||||
// 将毫秒转换为天数
|
||||
long days = diffInMillies / (1000 * 60 * 60 * 24); // 使用long来计算天数
|
||||
// 只查询正式账号
|
||||
List<QueryParam> queryParams = Arrays.asList(new QueryParam("isTest", 0));
|
||||
List<ApiUser> apiUsers = this.apiUserService.findApiUsersByParamsSortDate(queryParams);
|
||||
// 获取正式账号的ids
|
||||
List<Integer> ids = new ArrayList<Integer>();
|
||||
for (ApiUser apiUser : apiUsers) {
|
||||
ids.add((int) apiUser.getId());
|
||||
}
|
||||
// 只获取正式用户的
|
||||
List<OpenUriAnaly> analyAll = openUriAnalyService.findOpenUriAnalyLogs(model.getUserId(), null, startDate, endDate);
|
||||
List<OpenUriAnaly> analys = new ArrayList<OpenUriAnaly>();
|
||||
for (OpenUriAnaly analy : analyAll) {
|
||||
if (ids.contains(analy.getApiUserId())) {
|
||||
analys.add(analy);
|
||||
}
|
||||
}
|
||||
List<Integer> userIds = new ArrayList<Integer>();
|
||||
// 获取有访问记录的用户ID
|
||||
for (OpenUriAnaly analy : analys) {
|
||||
if (!userIds.contains(analy.getApiUserId())) {
|
||||
userIds.add(analy.getApiUserId());
|
||||
}
|
||||
}
|
||||
List<StatAnalyBo> bosAll = new ArrayList<StatAnalyBo>();
|
||||
if (CollectionUtils.isNotEmpty(userIds)) {
|
||||
// 根据访问遍历用户
|
||||
for (Integer userId : userIds) {
|
||||
long id = (long) userId;
|
||||
// 获取每个用户的访问
|
||||
List<OpenUriAnaly> analyUser = openUriAnalyService.findOpenUriAnalyLogs(id, null, startDate, endDate);
|
||||
ApiUserAuthority authority = apiUserAuthorityService.findByUserId(id);
|
||||
Map<String, Integer> countMap = authority.getTotalCountMap();
|
||||
// 获取 deadline
|
||||
ApiUser apiUser = apiUserService.findApiUserById(id);
|
||||
// 获取用户名
|
||||
User user = userService.findById(id);
|
||||
List<StatAnalyBo> bos = null;
|
||||
bos = createAnalyBosByLink(analyUser);
|
||||
long date = DateUtil.parseStringToDate("2016-11-23").getTime();
|
||||
if (startDate.getTime() >= date && endDate.getTime() > date) {
|
||||
List<ApiInvokeStatistical> apiInvokeStatisticalList = apiInvokeStatisticalService.findApiInvokeStatisticalList(id, startDate, endDate);
|
||||
if (bos != null && apiInvokeStatisticalList.size() > 0) {
|
||||
Map<String, Map<String, Integer>> resultMap = getCountByUri(apiInvokeStatisticalList);
|
||||
for (StatAnalyBo boTemp : bos) {
|
||||
boTemp.setPurpose(apiUser.getPurpose());
|
||||
String url = boTemp.getKey();
|
||||
boTemp.setSuccessEffectiveTime(resultMap.get(url).get("successEffectiveCount"));
|
||||
boTemp.setSuccessInvalidTime(resultMap.get(url).get("successInvalidCount"));
|
||||
boTemp.setDefeatTime(resultMap.get(url).get("defeatCount"));
|
||||
if (apiUser != null) {
|
||||
boTemp.setDeadline(apiUser.getDeadTime());
|
||||
}
|
||||
long curVistedCount = apiVisitCountService.findApiVisitCountByDay(id, url, DateUtil.truncateDay(new Date()));
|
||||
long historyCount = apiVisitCountService.findApiVisitAllCount(id, url, DateUtil.truncateDay(new Date()));
|
||||
if (countMap != null && countMap.containsKey(url)) {
|
||||
Integer totalCount = countMap.get(url);
|
||||
long usedCount = curVistedCount + historyCount;
|
||||
boTemp.setSurplusCallCount(Integer.valueOf((int) (totalCount - usedCount)));
|
||||
}
|
||||
if (user != null) {
|
||||
boTemp.setName(user.getDisplayName());
|
||||
boTemp.setUserId((int) user.getId());
|
||||
}
|
||||
|
||||
List<ApiVisitCount> apiVisitCounts = apiVisitCountService.findAllApiVisitCount(id, boTemp.getKey(), startDate, endDate);
|
||||
int total = 0;
|
||||
for (ApiVisitCount apiVisitCount : apiVisitCounts) {
|
||||
total += apiVisitCount.getVisitCount();
|
||||
}
|
||||
boTemp.setBillingCount(total);
|
||||
double dailyUse = boTemp.getCount() * 1.0 / days;
|
||||
boTemp.setDailyUse(dailyUse);
|
||||
}
|
||||
}
|
||||
}
|
||||
bosAll.addAll(bos);
|
||||
|
||||
}
|
||||
}
|
||||
this.buildReport(bosAll, response, startDate, endDate);
|
||||
model.setViewType("stream");
|
||||
return model;
|
||||
}
|
||||
|
||||
private void buildReport(List<StatAnalyBo> bos, HttpServletResponse response, Date startDate, Date endDate) {
|
||||
try {
|
||||
List<List<String>> dataList = new ArrayList<List<String>>();
|
||||
|
||||
for (StatAnalyBo bo : bos) {
|
||||
List<String> row = new ArrayList<String>();
|
||||
row.add(bo.getUserId() != null ? bo.getUserId().toString() : "");
|
||||
row.add(bo.getName() != null ? bo.getName().toString() : "");
|
||||
if (bo.getPurpose() != null) {
|
||||
Integer purpose = bo.getPurpose();
|
||||
if (purpose == 1) {
|
||||
row.add("API");
|
||||
} else if (purpose == 2) {
|
||||
row.add("PDS");
|
||||
} else if (purpose == 4) {
|
||||
row.add("房生活");
|
||||
}
|
||||
}else {
|
||||
row.add("");
|
||||
}
|
||||
row.add(bo.getKey() != null ? bo.getKey().toString() : "");
|
||||
row.add(bo.getSurplusCallCount() != null ? bo.getSurplusCallCount().toString() : "");
|
||||
row.add(bo.getDeadline() != null ? DateUtils.format(bo.getDeadline(), "yyyy/MM/dd") : "");
|
||||
row.add(bo.getCount() != null ? bo.getCount().toString() : "");
|
||||
if (bo.getDailyUse() !=null){
|
||||
// 格式化为保留一位小数的字符串
|
||||
String formattedDailyUse = String.format("%.1f", bo.getDailyUse());
|
||||
row.add(formattedDailyUse);
|
||||
}else {
|
||||
row.add("");
|
||||
}
|
||||
row.add(bo.getPercent() != null ? bo.getPercent().toString() : "");
|
||||
row.add(bo.getSuccessEffectiveTime() != null ? bo.getSuccessEffectiveTime().toString() : "");
|
||||
row.add(bo.getSuccessInvalidTime() != null ? bo.getSuccessInvalidTime().toString() : "");
|
||||
row.add(bo.getDefeatTime() != null ? bo.getDefeatTime().toString() : "");
|
||||
row.add(bo.getBillingCount() != null ? bo.getBillingCount().toString() : "");
|
||||
dataList.add(row);
|
||||
}
|
||||
|
||||
String fileName = "用户账号使用统计(" + DateUtils.format(startDate, "yyyy/MM/dd") + "—" + DateUtils.format(endDate, "yyyy/MM/dd") + ").xlsx";
|
||||
// ExcelWriter excelWriter = EasyExcel.write(response.getOutputStream()).build();
|
||||
String displayName = URLEncoder.encode(fileName, "UTF-8");
|
||||
response.setHeader("Content-Disposition", "attachment; filename=\"" + displayName + "\"");
|
||||
response.setContentType("application/x-download;charset=utf-8");
|
||||
Workbook workbook = new XSSFWorkbook();
|
||||
Sheet sheet = workbook.createSheet("用户账号使用统计");
|
||||
List<String> head = Arrays.asList(
|
||||
"用户ID", "账号", "账号用途", "接口", "剩余使用次数", "合同到期时间", "访问次数", "日均访问量",
|
||||
"占比(%)", "成功并有效", "成功并无效", "失败次数", "计费次数");
|
||||
// 创建表头
|
||||
Row headRow = sheet.createRow(0);
|
||||
for (int i = 0; i < head.size(); i++) {
|
||||
Cell cell = headRow.createCell(i);
|
||||
cell.setCellValue(head.get(i));
|
||||
}
|
||||
// 写入数据,从第二行
|
||||
for (int i = 0; i < dataList.size(); i++) {
|
||||
Row row = sheet.createRow(i + 1);
|
||||
List<String> data = dataList.get(i);
|
||||
for (int j = 0; j < data.size(); j++) {
|
||||
Cell cell = row.createCell(j);
|
||||
cell.setCellValue(data.get(j));
|
||||
}
|
||||
}
|
||||
ServletOutputStream outputStream = response.getOutputStream();
|
||||
workbook.write(outputStream);
|
||||
} catch (IOException e) {
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
List<List<String>> formatHead(String[] headArr) {
|
||||
List<List<String>> head = new ArrayList<List<String>>();
|
||||
for (String name : headArr) {
|
||||
List<String> temp = new ArrayList<String>();
|
||||
temp.add(name);
|
||||
head.add(temp);
|
||||
}
|
||||
return head;
|
||||
}
|
||||
|
||||
public Object userVisitStatAction(ApiUserModel model) throws IllegalAccessException {
|
||||
Date startDate = null;
|
||||
Date endDate = null;
|
||||
if (StringUtils.isNotBlank(model.getBegDate())) {
|
||||
@@ -1075,8 +1288,12 @@ public class ApiUserController {
|
||||
// bos = createAnalyBosByUser(analys);
|
||||
// model.setChartSeries(toChartString(bos));
|
||||
// }else if(model.getWay() == 2){
|
||||
Long userId = model.getUserId();
|
||||
ApiUserAuthority authority = apiUserAuthorityService.findByUserId(userId);
|
||||
Map<String, Integer> countMap = authority.getTotalCountMap();
|
||||
ApiUser apiUser = apiUserService.findApiUserById(userId);
|
||||
if (model.getWay() == 2) {
|
||||
User user = userService.findById(model.getUserId());
|
||||
User user = userService.findById(userId);
|
||||
if (user != null) {
|
||||
model.setUserName(user.getDisplayName());
|
||||
}
|
||||
@@ -1120,6 +1337,16 @@ public class ApiUserController {
|
||||
boTemp.setSuccessEffectiveTime(resultMap.get(url).get("successEffectiveCount"));
|
||||
boTemp.setSuccessInvalidTime(resultMap.get(url).get("successInvalidCount"));
|
||||
boTemp.setDefeatTime(resultMap.get(url).get("defeatCount"));
|
||||
if (apiUser != null) {
|
||||
boTemp.setDeadline(apiUser.getDeadTime());
|
||||
}
|
||||
long curVistedCount = apiVisitCountService.findApiVisitCountByDay(userId, url, DateUtil.truncateDay(new Date()));
|
||||
long historyCount = apiVisitCountService.findApiVisitAllCount(userId, url, DateUtil.truncateDay(new Date()));
|
||||
if (countMap != null && countMap.containsKey(url)) {
|
||||
Integer totalCount = countMap.get(url);
|
||||
long usedCount = curVistedCount + historyCount;
|
||||
boTemp.setSurplusCallCount(Integer.valueOf((int) (totalCount - usedCount)));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+4
@@ -74,6 +74,8 @@ public class InquiryController {
|
||||
role = fjtRoleService.findRolesByName("农行");
|
||||
}else if(2 == bkoffUsers.getSupportInquiryTask().intValue()){
|
||||
role = fjtRoleService.findRolesByName("深圳农行");
|
||||
}else if(3 == bkoffUsers.getSupportInquiryTask().intValue()){
|
||||
role = fjtRoleService.findRolesByName("大正评估");
|
||||
}
|
||||
|
||||
List<Long> zqUserIds = userCityService.findUserIdByCity("肇庆");
|
||||
@@ -87,6 +89,8 @@ public class InquiryController {
|
||||
}
|
||||
}else if(vo2.getRole()==16){
|
||||
userIds.add(vo2.getId());
|
||||
}else if(vo2.getRole()==18){
|
||||
userIds.add(vo2.getId());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -51,6 +51,7 @@
|
||||
<select name="isTest">
|
||||
<option value="0">正式账户</option>
|
||||
<option value="1">测试账户</option>
|
||||
<option value="2">内部账户</option>
|
||||
</select>
|
||||
</td>
|
||||
</tr>
|
||||
|
||||
@@ -12,9 +12,9 @@
|
||||
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
|
||||
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
|
||||
<style>
|
||||
|
||||
|
||||
label{display:inline-block}
|
||||
|
||||
|
||||
.main-content {
|
||||
padding-bottom: 0;
|
||||
}
|
||||
@@ -47,28 +47,51 @@
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<form action="<#if way?? && way==0>../apiUser/pandect<#else>../apiUser/apiVisitStat</#if>" method="post">
|
||||
<input type="hidden" name="way" value="${way!}"/>
|
||||
<#if way!=3>
|
||||
<label for="districtName" style="margin-left: 15px;">用户ID:</label>
|
||||
<input type="text" name="userId" value="${userId!}" class="input-medium" style="margin-top:9px;" ></input>
|
||||
<#else>
|
||||
<label for="districtName" style="margin-left: 15px;">城市:</label>
|
||||
<input type="text" class="input-medium" readonly="readonly" id="city" name="citys" value="${citys!}"/><input type="button" class="input-medium" id="butt" value="选择城市"/>
|
||||
</#if>
|
||||
<label for="startDate"> 日期:</label>
|
||||
<input placeholder="开始时间" class="form-control" style="width:140px;" name="begDate" id="begDate" value="<#if begDate??>${begDate}</#if>" onfocus="WdatePicker({dateFmt:'yyyy-MM-dd'})" />-
|
||||
<input id="endDate" placeholder="结束时间" class="form-control" style="width:140px;" name="endDate" value="${endDate!}" onfocus="WdatePicker({dateFmt:'yyyy-MM-dd'})"/>
|
||||
<form id="myForm" action="<#if way?? && way==0>../apiUser/pandect<#else>../apiUser/apiVisitStat</#if>" method="post">
|
||||
<input type="hidden" name="way" value="${way!}"/>
|
||||
|
||||
|
||||
<input id="queryBtn" class ="btn btn-primary" type="submit" value="查询" />
|
||||
<#if way!=3>
|
||||
<label for="districtName" style="margin-left: 15px;">用户ID:</label>
|
||||
<input type="text" name="userId" value="${userId!}" class="input-medium" style="margin-top:9px;" />
|
||||
<#else>
|
||||
<label for="districtName" style="margin-left: 15px;">城市:</label>
|
||||
<input type="text" class="input-medium" readonly="readonly" id="city" name="citys" value="${citys!}"/>
|
||||
<input type="button" class="input-medium" id="butt" value="选择城市"/>
|
||||
</#if>
|
||||
|
||||
<#if way!=3>
|
||||
<label for="">(备注:成功有效、成功无效、失败的次数统计从2016-11-23开始)</label>
|
||||
</#if>
|
||||
</form>
|
||||
<label for="startDate"> 日期:</label>
|
||||
|
||||
<!-- 下拉菜单选择最近一周、最近一月或自定义 -->
|
||||
<select id="dateRange" class="form-control" style="width:160px;" onchange="updateDateRange(this.value)">
|
||||
<option value="custom">自定义</option>
|
||||
<option value="lastWeek">最近一周</option>
|
||||
<option value="lastMonth">最近一月</option>
|
||||
</select>
|
||||
|
||||
<!-- 自定义日期输入框 -->
|
||||
<input placeholder="开始时间" class="form-control" style="width:140px;" name="begDate" id="begDate"
|
||||
value="<#if begDate??>${begDate}</#if>"
|
||||
onfocus="WdatePicker({dateFmt:'yyyy-MM-dd'})" />
|
||||
|
||||
-
|
||||
|
||||
<input id="endDate" placeholder="结束时间" class="form-control" style="width:140px;" name="endDate"
|
||||
value="${endDate!}"
|
||||
onfocus="WdatePicker({dateFmt:'yyyy-MM-dd'})" />
|
||||
|
||||
<!-- 查询按钮 -->
|
||||
<input id="queryBtn" class="btn btn-primary" type="submit" value="查询" onclick="submitForm('query')" />
|
||||
|
||||
<!-- 下载按钮 -->
|
||||
<input id="downloadBtn" class="btn btn-secondary" type="submit" value="导出" onclick="submitForm('download')" />
|
||||
|
||||
<#if way!=3>
|
||||
<label for="">(备注:成功有效、成功无效、失败的次数统计从2016-11-23开始)</label>
|
||||
</#if>
|
||||
</form>
|
||||
<#--<input id="expect" class ="btn btn-primary" type="submit" value="导出" />-->
|
||||
<a class="btn common_btn" href="importExcel?city=${citys!}&begDate=${begDate!}&endDate=${endDate!}">导出</a>
|
||||
<#-- <a class="btn common_btn" href="importExcel?city=${citys!}&begDate=${begDate!}&endDate=${endDate!}">导出</a>-->
|
||||
|
||||
|
||||
<div id="isHide" style="display: none">
|
||||
<ul>
|
||||
@@ -544,7 +567,7 @@
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<#if statAnalyBoList??>
|
||||
<#if statAnalyBoList??>
|
||||
<#list statAnalyBoList as bo>
|
||||
<tr>
|
||||
<#if way?? && way==1>
|
||||
@@ -591,7 +614,7 @@
|
||||
<#else>
|
||||
<tr><td ></td></tr>
|
||||
</#if>
|
||||
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
<div class="tabbable" style="padding-top: 60px;">
|
||||
@@ -643,7 +666,7 @@
|
||||
$("#statBut3").removeClass("cur");
|
||||
|
||||
});
|
||||
|
||||
|
||||
$("#statBut2").click(function(){
|
||||
this.addClass("cur");
|
||||
$("#statBut1").removeClass("cur");
|
||||
@@ -697,8 +720,8 @@
|
||||
}
|
||||
})
|
||||
});
|
||||
|
||||
|
||||
|
||||
|
||||
$(function () {
|
||||
|
||||
$('#container').highcharts({
|
||||
@@ -734,8 +757,46 @@
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
</script>
|
||||
function updateDateRange(option) {
|
||||
const now = new Date();
|
||||
const begDateInput = document.getElementById('begDate');
|
||||
const endDateInput = document.getElementById('endDate');
|
||||
|
||||
if (option === 'lastWeek') {
|
||||
const lastWeek = new Date();
|
||||
lastWeek.setDate(now.getDate() - 7);
|
||||
begDateInput.value = formatDate(lastWeek);
|
||||
endDateInput.value = formatDate(now);
|
||||
} else if (option === 'lastMonth') {
|
||||
const lastMonth = new Date();
|
||||
lastMonth.setMonth(now.getMonth() - 1);
|
||||
begDateInput.value = formatDate(lastMonth);
|
||||
endDateInput.value = formatDate(now);
|
||||
} else {
|
||||
begDateInput.value = '';
|
||||
endDateInput.value = '';
|
||||
}
|
||||
}
|
||||
|
||||
function formatDate(date) {
|
||||
const year = date.getFullYear();
|
||||
const month = String(date.getMonth() + 1).padStart(2, '0');
|
||||
const day = String(date.getDate()).padStart(2, '0');
|
||||
return year + '-' + month + '-' + day;
|
||||
}
|
||||
function submitForm(action) {
|
||||
const form = document.getElementById('myForm');
|
||||
if (action === 'query') {
|
||||
debugger
|
||||
form.action = "<#if way?? && way==0>../apiUser/pandect<#else>../apiUser/apiVisitStat</#if>";
|
||||
} else if (action === 'download') {
|
||||
form.action = "../apiUser/export";
|
||||
}
|
||||
form.submit();
|
||||
}
|
||||
|
||||
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
</#compress>
|
||||
|
||||
@@ -61,12 +61,18 @@
|
||||
<td>账户类型</td>
|
||||
<td>
|
||||
<select name="isTest">
|
||||
<#if apiUserBo.isTest==0>
|
||||
<#if apiUserBo.isTest == 0>
|
||||
<option value="0" selected="selected">正式账户</option>
|
||||
<option value="1">测试账户</option>
|
||||
<#else>
|
||||
<option value="0"> 正式账户</option>
|
||||
<option value="2">内部账户</option>
|
||||
<#elseif apiUserBo.isTest == 1>
|
||||
<option value="0">正式账户</option>
|
||||
<option value="1" selected="selected">测试账户</option>
|
||||
<option value="2">内部账户</option>
|
||||
<#else>
|
||||
<option value="0">正式账户</option>
|
||||
<option value="1">测试账户</option>
|
||||
<option value="2" selected="selected">内部账户</option>
|
||||
</#if>
|
||||
</select>
|
||||
</td>
|
||||
|
||||
@@ -131,8 +131,10 @@
|
||||
<td align="center">
|
||||
<#if bo.isTest==0>
|
||||
正式账户
|
||||
<#else>
|
||||
<#elseif bo.isTest==1>
|
||||
测试账户
|
||||
<#else>
|
||||
内部账户
|
||||
</#if>
|
||||
</td>
|
||||
<td>
|
||||
|
||||
@@ -91,6 +91,12 @@ table tr th {
|
||||
<#if query=="uri">URL<#elseif query=="day">日期<#else>访问时长</#if>
|
||||
</#if>
|
||||
</th>
|
||||
<th width="9%" colspan="1">
|
||||
剩余使用次数
|
||||
</th>
|
||||
<th width="9%" colspan="1">
|
||||
合同到期时间
|
||||
</th>
|
||||
<th width="9%" colspan="1">
|
||||
访问次数
|
||||
</th>
|
||||
@@ -145,6 +151,8 @@ table tr th {
|
||||
</#if>
|
||||
</#if>
|
||||
</td>
|
||||
<td width="9%"><#if bo.surplusCallCount??>${bo.surplusCallCount}</#if></td>
|
||||
<td width="9%"><#if bo.deadline??>${bo.deadline?string("yyyy-MM-dd")}</#if></td>
|
||||
<td width="9%">${bo.count}</td>
|
||||
<td width="9%">${bo.percent}</td>
|
||||
<#if way==2 && query=='uri'><td>${resourceMap[bo.key]!}</td></#if>
|
||||
|
||||
@@ -176,7 +176,7 @@ table{text-align: left;}
|
||||
}
|
||||
})
|
||||
|
||||
if(role==12||role==17){
|
||||
if(role==12||role==17||role==18){
|
||||
if(userStr.length>0){
|
||||
console.log(userStr);
|
||||
$("input[name=roleUsers]").val(userStr);
|
||||
@@ -306,7 +306,7 @@ $("input[name=email]").blur(function(){
|
||||
$("#role").change(function() {
|
||||
var role = $(this).val();
|
||||
//只对农行生效
|
||||
if (role == 12 || role == 17) {
|
||||
if (role == 12 || role == 17 || role == 18) {
|
||||
$("#show_chk_all").removeAttr("style");
|
||||
jQuery.ajax({
|
||||
type: 'POST',
|
||||
@@ -340,7 +340,7 @@ $("input[name=email]").blur(function(){
|
||||
}
|
||||
return false;
|
||||
}
|
||||
<#if user?? && (user.role == 12 || user.role == 17)>
|
||||
<#if user?? && (user.role == 12 || user.role == 17|| user.role == 18)>
|
||||
$("#show_chk_all").removeAttr("style");
|
||||
jQuery.ajax({
|
||||
type: 'POST',
|
||||
|
||||
@@ -7,7 +7,7 @@ public interface Constants {
|
||||
String FANGJIA_CRAWL_MONGO_URI = "mongodb://192.168.0.235:27018";
|
||||
String LOG_MONGO_URI = "mongodb://192.168.0.42:52002";
|
||||
String SEABIRD_MONGO_URI = "mongodb://192.168.0.74:57007";
|
||||
String SEAWATER_MONGO_URI = "mongodb://192.168.0.206:27020";
|
||||
String SEAWATER_MONGO_URI = "mongodb://root:Goojia%40206!!#@192.168.0.206:27020";
|
||||
String MSVC_FANG_JIA_DATA_SERVICE_BASE_PATH ="http://192.168.0.100:38080/FangJiaDataService";
|
||||
|
||||
}
|
||||
|
||||
@@ -15,15 +15,15 @@ import com.fangjia.dal.MongoEntity;
|
||||
public class SqmetreRent implements MongoEntity {
|
||||
|
||||
/**
|
||||
*
|
||||
*
|
||||
*/
|
||||
private static final long serialVersionUID = -5525712882803653336L;
|
||||
private String id;
|
||||
private String city;
|
||||
private String region;
|
||||
private String districtName;
|
||||
private Integer avgPrice;
|
||||
private Integer calcAvgPrice;
|
||||
private Number avgPrice;
|
||||
private Number calcAvgPrice;
|
||||
private Double markup;
|
||||
private Date weekend;
|
||||
private String cat;
|
||||
@@ -74,20 +74,20 @@ public class SqmetreRent implements MongoEntity {
|
||||
}
|
||||
|
||||
@Column(name = "avg_price")
|
||||
public Integer getAvgPrice() {
|
||||
public Number getAvgPrice() {
|
||||
return avgPrice;
|
||||
}
|
||||
|
||||
public void setAvgPrice(Integer avgPrice) {
|
||||
public void setAvgPrice(Number avgPrice) {
|
||||
this.avgPrice = avgPrice;
|
||||
}
|
||||
|
||||
@Column(name = "calc_avg_price")
|
||||
public Integer getCalcAvgPrice() {
|
||||
public Number getCalcAvgPrice() {
|
||||
return calcAvgPrice;
|
||||
}
|
||||
|
||||
public void setCalcAvgPrice(Integer calcAvgPrice) {
|
||||
public void setCalcAvgPrice(Number calcAvgPrice) {
|
||||
this.calcAvgPrice = calcAvgPrice;
|
||||
}
|
||||
|
||||
@@ -163,7 +163,5 @@ public class SqmetreRent implements MongoEntity {
|
||||
public void setCat(String cat) {
|
||||
this.cat = cat;
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ public class ApiUser implements SqlEntity {
|
||||
private Date creationDate;
|
||||
private Date modificationDate;
|
||||
private String company;
|
||||
private int isTest; //0-正式客戶 1-测试用户
|
||||
private int isTest; //0-正式客戶 1-测试用户 2-内部测试用户
|
||||
private Date deadTime;
|
||||
/**
|
||||
* 账户用途
|
||||
|
||||
+79
-3
@@ -32,6 +32,7 @@ import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.impl.conn.DefaultProxyRoutePlanner;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.sothis.core.util.CollectionUtils;
|
||||
import org.sothis.core.util.Pager;
|
||||
import sun.misc.BASE64Encoder;
|
||||
|
||||
@@ -42,6 +43,7 @@ import java.net.PasswordAuthentication;
|
||||
import java.net.Proxy;
|
||||
import java.nio.charset.Charset;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
@@ -439,6 +441,8 @@ public class AddrSvcController {
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else if (!"2".equals(type)) { // 对私的不用齐梁
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
try {
|
||||
//如果是江苏省或者浙江省 调用齐梁的结果
|
||||
@@ -450,6 +454,12 @@ public class AddrSvcController {
|
||||
Map<String, String> map = senRequest(address, qltype);
|
||||
// 涉农结果不一样
|
||||
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
|
||||
// 齐梁县政府但是涉农为农,取我们的结果 扬中市三茅街道中华村永固河路1号
|
||||
if ("是".equals(map.get("countyGov")) && "农".equals(map.get("verifyFlag"))) {
|
||||
basicModel.setResult(result);
|
||||
mav.setModel(basicModel);
|
||||
return mav;
|
||||
}
|
||||
//城市不一样 用我们
|
||||
if (result.containsKey("city")) {
|
||||
if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
|
||||
@@ -457,11 +467,15 @@ public class AddrSvcController {
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
|
||||
@@ -488,6 +502,45 @@ public class AddrSvcController {
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断县级市
|
||||
*/
|
||||
public String countyLevelCity(Map<String, String> map) {
|
||||
if (map.get("city") == null) {
|
||||
return "";
|
||||
}
|
||||
if (map.get("city").endsWith("区")) {
|
||||
return "否";
|
||||
}
|
||||
if (map.get("city").equals("东莞市") || map.get("city").equals("嘉峪关市") || map.get("city").equals("中山市") || map.get("city").equals("儋州市") ) {
|
||||
return "否";
|
||||
}
|
||||
return "是";
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断所属区域
|
||||
*
|
||||
* @param map
|
||||
* @return
|
||||
*/
|
||||
public String addrArea(Map<String, String> map, String type) {
|
||||
if (map.get("verifyFlag") == null ) {
|
||||
return "";
|
||||
}
|
||||
if (map.get("verifyFlag").equals("非农")) {
|
||||
return "城市区域";
|
||||
}
|
||||
if (map.get("verifyFlag").equals("农")) {
|
||||
return "农村区域";
|
||||
}
|
||||
if ("2".equals(type) && "是".equals(map.get("countyGov"))) {
|
||||
return "县城城区";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* 涉农识别-新规则-上海征信
|
||||
*
|
||||
@@ -530,7 +583,8 @@ public class AddrSvcController {
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else if (!"2".equals(type)) { // 对私的不用齐梁
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
try {
|
||||
//如果是江苏省或者浙江省 调用齐梁的结果
|
||||
@@ -542,17 +596,27 @@ public class AddrSvcController {
|
||||
Map<String, String> map = senRequest(address, qltype);
|
||||
// 涉农结果不一样
|
||||
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
|
||||
// 齐梁县政府但是涉农为农,取我们的结果 扬中市三茅街道中华村永固河路1号
|
||||
if ("是".equals(map.get("countyGov")) && "农".equals(map.get("verifyFlag"))) {
|
||||
basicModel.setResult(result);
|
||||
mav.setModel(basicModel);
|
||||
return mav;
|
||||
}
|
||||
//城市不一样 用我们
|
||||
if (result.containsKey("city")) {
|
||||
if ((!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
@@ -607,7 +671,8 @@ public class AddrSvcController {
|
||||
params.put("type", type);
|
||||
params.put("showFullValue", "2");
|
||||
Pager pager = new Pager(1, 1);
|
||||
VillageIdentify villageResult = villageIdentifyService.findAll(villageType, model.getCity(), model.getAddress(), pager).get(0);
|
||||
List<VillageIdentify> villageIdentifyList = villageIdentifyService.findByCityAndAddress(villageType, model.getCity(), model.getAddress(), pager);
|
||||
VillageIdentify villageResult = CollectionUtils.isNotEmpty(villageIdentifyList) ? villageIdentifyList.get(0) : null;
|
||||
if (villageResult != null && StringUtils.isNotBlank(villageResult.getVerifyFlag())) {
|
||||
Map<String, String> resultMap = new HashMap<String, String>();
|
||||
resultMap.put("province",villageResult.getProvince());
|
||||
@@ -659,7 +724,8 @@ public class AddrSvcController {
|
||||
//我们省级及以上有开发区,直接用我们的结果
|
||||
if (result.containsKey("devDistrictName")) {
|
||||
basicModel.setResult(result);
|
||||
|
||||
} else if (!"2".equals(type)) { // 对私的不用齐梁
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
try {
|
||||
//如果是江苏省或者浙江省 调用齐梁的结果
|
||||
@@ -672,17 +738,27 @@ public class AddrSvcController {
|
||||
Map<String, String> map = senRequest(address, qltype);
|
||||
// 涉农结果不一样
|
||||
if (!map.get("verifyFlag").equals(result.get("verifyFlag"))) {
|
||||
// 齐梁县政府但是涉农为农,取我们的结果 扬中市三茅街道中华村永固河路1号
|
||||
if ("是".equals(map.get("countyGov")) && "农".equals(map.get("verifyFlag"))) {
|
||||
basicModel.setResult(result);
|
||||
mav.setModel(basicModel);
|
||||
return mav;
|
||||
}
|
||||
//有市辖建制镇, 或者城市不一样 用我们
|
||||
if (result.containsKey("city")) {
|
||||
if ("是".equals(result.get("administrativeTowns")) || (!result.get("city").equals(map.get("city")) && StringUtils.isNotBlank(result.getString("city")))) {
|
||||
basicModel.setResult(result);
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
} else {
|
||||
map.put("addrVerify", "地址有效");
|
||||
map.put("addrArea", addrArea(map, type));
|
||||
map.put("countyLevelCity", countyLevelCity(map));
|
||||
map.put("typeName", areaMap.get(map.get("typeCode")));
|
||||
basicModel.setResult(map);
|
||||
}
|
||||
|
||||
+5671
-5615
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -203,7 +203,7 @@ public class RentController {
|
||||
private Double getSqmetreRent(District district){
|
||||
SqmetreRent sqmetreRent = sqmetreRentService.findNearestSqmetreRent(district.getCity(), district.getRegion(), district.getName(), null, "district");
|
||||
if(null != sqmetreRent){
|
||||
return Double.valueOf(sqmetreRent.getAvgPrice());
|
||||
return Double.valueOf(sqmetreRent.getAvgPrice().intValue());
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
+176
-98
@@ -1,24 +1,32 @@
|
||||
package com.fangjia.open.controllers;
|
||||
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.*;
|
||||
|
||||
import com.fangjia.core.util.AnalyticsUtil;
|
||||
import com.fangjia.core.util.BeanUtils;
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.mongo.model.SendSmsLog;
|
||||
import com.fangjia.dal.mongo.open.model.ApiUserAuthority;
|
||||
import com.fangjia.dal.mongo.open.model.OpenResource;
|
||||
import com.fangjia.dal.mongo.tool.model.OpenUriAnaly;
|
||||
import com.fangjia.open.HttpStatusCodeEnum;
|
||||
import com.fangjia.open.interceptors.AuthenticationBean;
|
||||
import com.fangjia.open.model.*;
|
||||
import com.fangjia.open.model.weixin.SendCodeJsonModel;
|
||||
import com.fangjia.open.model.weixin.SendCodeModel;
|
||||
import com.fangjia.service.open.*;
|
||||
import net.sf.json.JSONObject;
|
||||
|
||||
import org.apache.commons.codec.digest.DigestUtils;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.sothis.web.mvc.ActionContext;
|
||||
import org.sothis.web.mvc.ActionInvocation;
|
||||
import org.sothis.web.mvc.annotation.Param;
|
||||
|
||||
import com.fangjia.dal.sql.model.User;
|
||||
import com.fangjia.open.HttpCode;
|
||||
import com.fangjia.open.OpenHttpCodeEnum;
|
||||
import com.fangjia.open.model.RestfulModelAndView;
|
||||
import com.fangjia.open.model.UsersModel;
|
||||
import com.fangjia.service.mongo.mobile.UserInfoService;
|
||||
import com.fangjia.service.sql.UserService;
|
||||
import com.fangjia.service.util.UsersUtils;
|
||||
@@ -26,100 +34,170 @@ import com.fangjia.service.util.UsersUtils;
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
|
||||
public class UsersController {
|
||||
private UserService userService;
|
||||
private UserInfoService userInfoService;
|
||||
private UserService userService;
|
||||
private UserInfoService userInfoService;
|
||||
|
||||
/**
|
||||
*
|
||||
* @param model
|
||||
* @return 1 无此用户 2 非法请求 3 密码错误 0 正常登陆
|
||||
* @throws Exception
|
||||
*/
|
||||
public Object loginAction(UsersModel model) throws Exception {
|
||||
Map<String, Object> infomain = new HashMap<String, Object>();
|
||||
if (StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getPwd())) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.EINVAL.ordinal());
|
||||
} else {
|
||||
User user = userService.findByUserName(model.getName());
|
||||
if (null == user) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.NOUSER.ordinal());
|
||||
} else if (!user.getPassword().equals(DigestUtils.md5Hex(model.getPwd()))) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.ERRPWD.ordinal());
|
||||
} else {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.SUCCESS.ordinal());
|
||||
infomain.put("displayname", user.getDisplayName());
|
||||
infomain.put("username", user.getUserName());
|
||||
infomain.put("id", user.getId());
|
||||
}
|
||||
}
|
||||
JSONObject json = JSONObject.fromObject(infomain);
|
||||
return json.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param model
|
||||
* @return 1 无此用户 2 非法请求 3 原始密码错误 0 修改成功
|
||||
* @throws Exception
|
||||
*/
|
||||
public Object modifyPwdAction(UsersModel model) throws Exception {
|
||||
Map<String, Object> infomain = new HashMap<String, Object>();
|
||||
if (StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getPwd())) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.EINVAL.ordinal());
|
||||
} else {
|
||||
User user = userService.findByUserName(model.getName());
|
||||
if (null == user) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.NOUSER.ordinal());
|
||||
} else if (!user.getPassword().equals(DigestUtils.md5Hex(model.getPwd()))) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.ERRPWD.ordinal());
|
||||
} else {
|
||||
user.setPassword(DigestUtils.md5Hex(model.getNewPwd()));
|
||||
userService.updateUser(user);
|
||||
infomain.put("flag", UsersUtils.LoginEnum.SUCCESS.ordinal());
|
||||
}
|
||||
}
|
||||
JSONObject json = JSONObject.fromObject(infomain);
|
||||
return json.toString();
|
||||
}
|
||||
|
||||
public Object saveAction(@Param(name = "ime") String ime, @Param(name = "name") String name, @Param(name = "description") String description, @Param(name = "image") String image, @Param(name = "type") String type) {
|
||||
|
||||
if (ime == null || "".equals(ime.trim())) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "手机唯一标示缺失!"));
|
||||
return mav;
|
||||
}
|
||||
if (name == null || "".equals(name.trim())) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "用户名缺失!"));
|
||||
return mav;
|
||||
}
|
||||
if (type == null || "".equals(type.trim())) {
|
||||
type = "default";
|
||||
}
|
||||
|
||||
com.fangjia.dal.mongo.mobile.model.User userInfo = userInfoService.save(ime, name, description, image, type);
|
||||
|
||||
Map<String, Object> infomain = new HashMap<String, Object>();
|
||||
infomain.put("uid", userInfo.getId());
|
||||
infomain.put("ime", userInfo.getIme());
|
||||
infomain.put("name", userInfo.getName());
|
||||
infomain.put("type", userInfo.getType());
|
||||
infomain.put("description", userInfo.getDescription());
|
||||
infomain.put("image", userInfo.getImage());
|
||||
JSONObject json = JSONObject.fromObject(infomain);
|
||||
return json.toString();
|
||||
}
|
||||
|
||||
public void setUserService(UserService userService) {
|
||||
this.userService = userService;
|
||||
}
|
||||
|
||||
public void setUserInfoService(UserInfoService userInfoService) {
|
||||
this.userInfoService = userInfoService;
|
||||
}
|
||||
private ApiVisitCountService apiVisitCountService;
|
||||
private ApiUserAuthorityService apiUserAuthorityService;
|
||||
private OpenResourceService openResourceService;
|
||||
private OpenUriAnalyService openUriAnalyService;
|
||||
|
||||
|
||||
/**
|
||||
* @param model
|
||||
* @return 1 无此用户 2 非法请求 3 密码错误 0 正常登陆
|
||||
* @throws Exception
|
||||
*/
|
||||
public Object loginAction(UsersModel model) throws Exception {
|
||||
Map<String, Object> infomain = new HashMap<String, Object>();
|
||||
if (StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getPwd())) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.EINVAL.ordinal());
|
||||
} else {
|
||||
User user = userService.findByUserName(model.getName());
|
||||
if (null == user) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.NOUSER.ordinal());
|
||||
} else if (!user.getPassword().equals(DigestUtils.md5Hex(model.getPwd()))) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.ERRPWD.ordinal());
|
||||
} else {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.SUCCESS.ordinal());
|
||||
infomain.put("displayname", user.getDisplayName());
|
||||
infomain.put("username", user.getUserName());
|
||||
infomain.put("id", user.getId());
|
||||
}
|
||||
}
|
||||
JSONObject json = JSONObject.fromObject(infomain);
|
||||
return json.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* @param model
|
||||
* @return 1 无此用户 2 非法请求 3 原始密码错误 0 修改成功
|
||||
* @throws Exception
|
||||
*/
|
||||
public Object modifyPwdAction(UsersModel model) throws Exception {
|
||||
Map<String, Object> infomain = new HashMap<String, Object>();
|
||||
if (StringUtils.isBlank(model.getName()) || StringUtils.isBlank(model.getPwd())) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.EINVAL.ordinal());
|
||||
} else {
|
||||
User user = userService.findByUserName(model.getName());
|
||||
if (null == user) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.NOUSER.ordinal());
|
||||
} else if (!user.getPassword().equals(DigestUtils.md5Hex(model.getPwd()))) {
|
||||
infomain.put("flag", UsersUtils.LoginEnum.ERRPWD.ordinal());
|
||||
} else {
|
||||
user.setPassword(DigestUtils.md5Hex(model.getNewPwd()));
|
||||
userService.updateUser(user);
|
||||
infomain.put("flag", UsersUtils.LoginEnum.SUCCESS.ordinal());
|
||||
}
|
||||
}
|
||||
JSONObject json = JSONObject.fromObject(infomain);
|
||||
return json.toString();
|
||||
}
|
||||
|
||||
public Object saveAction(@Param(name = "ime") String ime, @Param(name = "name") String name, @Param(name = "description") String description, @Param(name = "image") String image, @Param(name = "type") String type) {
|
||||
|
||||
if (ime == null || "".equals(ime.trim())) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "手机唯一标示缺失!"));
|
||||
return mav;
|
||||
}
|
||||
if (name == null || "".equals(name.trim())) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "用户名缺失!"));
|
||||
return mav;
|
||||
}
|
||||
if (type == null || "".equals(type.trim())) {
|
||||
type = "default";
|
||||
}
|
||||
|
||||
com.fangjia.dal.mongo.mobile.model.User userInfo = userInfoService.save(ime, name, description, image, type);
|
||||
|
||||
Map<String, Object> infomain = new HashMap<String, Object>();
|
||||
infomain.put("uid", userInfo.getId());
|
||||
infomain.put("ime", userInfo.getIme());
|
||||
infomain.put("name", userInfo.getName());
|
||||
infomain.put("type", userInfo.getType());
|
||||
infomain.put("description", userInfo.getDescription());
|
||||
infomain.put("image", userInfo.getImage());
|
||||
JSONObject json = JSONObject.fromObject(infomain);
|
||||
return json.toString();
|
||||
}
|
||||
|
||||
public Object surplusCallCountAction() throws IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
ActionContext context = ActionContext.getContext();
|
||||
HttpServletRequest request = context.getRequest();
|
||||
AuthenticationBean instance = (AuthenticationBean) request.getAttribute(AuthenticationBean.ATTR_AUTHENTICATION_BEAN);
|
||||
if (null != instance) {
|
||||
User user = instance.user;
|
||||
List<OpenResource> openResourceList = openResourceService.findAll();
|
||||
List<ApiMonitorInfo> list = new ArrayList<ApiMonitorInfo>();
|
||||
long id = user.getId();
|
||||
ApiUserAuthority authority = apiUserAuthorityService.findByUserId(id);
|
||||
if (null != authority) {
|
||||
// 用户有的uri
|
||||
List<OpenResource> userResources = new ArrayList<OpenResource>();
|
||||
for (OpenResource openResource : openResourceList) {
|
||||
if (authority.getOpenResourseUris().contains(openResource.getUri())) {
|
||||
userResources.add(openResource);
|
||||
}
|
||||
}
|
||||
Map<String, Integer> countMap = authority.getTotalCountMap();
|
||||
if (null != countMap) {
|
||||
// 改成遍历用户有的uri
|
||||
for (OpenResource openResource : userResources) {
|
||||
String uri = openResource.getUri();
|
||||
ApiMonitorInfo info = new ApiMonitorInfo();
|
||||
info.setApiUri(uri);
|
||||
info.setApiName(openResource.getName());
|
||||
info.setUserId((int) id);
|
||||
info.setUsername(user.getUserName());
|
||||
if (null != countMap.get(uri) && countMap.get(uri) > 0) {
|
||||
long curVistedCount = apiVisitCountService.findApiVisitCountByDay(id, uri, DateUtil.truncateDay(new Date()));
|
||||
long historyCount = apiVisitCountService.findApiVisitAllCount(id, uri, DateUtil.truncateDay(new Date()));
|
||||
Integer totalCount = countMap.get(uri);
|
||||
info.setTotalCallCount(Long.valueOf(totalCount));
|
||||
long usedCount = curVistedCount + historyCount;
|
||||
info.setSurplusCallCount(totalCount - usedCount);
|
||||
info.setUsedCallCount(usedCount);
|
||||
}
|
||||
list.add(info);
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
BasicModel basicModel = new BasicModel(200, "ok");
|
||||
basicModel.setResult(list);
|
||||
mav.setModel(basicModel);
|
||||
} else {
|
||||
mav.setModel(new BasicModel(HttpStatusCodeEnum.BAD_REQUEST_ERROR_KEY));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
public void setUserService(UserService userService) {
|
||||
this.userService = userService;
|
||||
}
|
||||
|
||||
public void setUserInfoService(UserInfoService userInfoService) {
|
||||
this.userInfoService = userInfoService;
|
||||
}
|
||||
|
||||
public void setApiVisitCountService(ApiVisitCountService apiVisitCountService) {
|
||||
this.apiVisitCountService = apiVisitCountService;
|
||||
}
|
||||
|
||||
public void setApiUserAuthorityService(ApiUserAuthorityService apiUserAuthorityService) {
|
||||
this.apiUserAuthorityService = apiUserAuthorityService;
|
||||
}
|
||||
|
||||
public void setOpenResourceService(OpenResourceService openResourceService) {
|
||||
this.openResourceService = openResourceService;
|
||||
}
|
||||
|
||||
public void setOpenUriAnalyService(OpenUriAnalyService openUriAnalyService) {
|
||||
this.openUriAnalyService = openUriAnalyService;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
package com.fangjia.open.model;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class ApiMonitorInfo {
|
||||
private Integer userId;
|
||||
private String username;
|
||||
private Integer callCount;
|
||||
private Date callTime;
|
||||
private String apiUri;
|
||||
private String apiName;
|
||||
private Long surplusCallCount;
|
||||
private Long totalCallCount;
|
||||
private Long usedCallCount;
|
||||
private Date endTime;
|
||||
|
||||
public Long getSurplusCallCount() {
|
||||
return surplusCallCount;
|
||||
}
|
||||
public void setSurplusCallCount(Long surplusCallCount) {
|
||||
this.surplusCallCount = surplusCallCount;
|
||||
}
|
||||
public Date getEndTime() {
|
||||
return endTime;
|
||||
}
|
||||
public void setEndTime(Date endTime) {
|
||||
this.endTime = endTime;
|
||||
}
|
||||
public Integer getUserId() {
|
||||
return userId;
|
||||
}
|
||||
public void setUserId(Integer userId) {
|
||||
this.userId = userId;
|
||||
}
|
||||
public String getUsername() {
|
||||
return username;
|
||||
}
|
||||
public void setUsername(String username) {
|
||||
this.username = username;
|
||||
}
|
||||
public Integer getCallCount() {
|
||||
return callCount;
|
||||
}
|
||||
public void setCallCount(Integer callCount) {
|
||||
this.callCount = callCount;
|
||||
}
|
||||
public Date getCallTime() {
|
||||
return callTime;
|
||||
}
|
||||
public void setCallTime(Date callTime) {
|
||||
this.callTime = callTime;
|
||||
}
|
||||
public String getApiUri() {
|
||||
return apiUri;
|
||||
}
|
||||
public void setApiUri(String apiUri) {
|
||||
this.apiUri = apiUri;
|
||||
}
|
||||
public String getApiName() {
|
||||
return apiName;
|
||||
}
|
||||
public void setApiName(String apiName) {
|
||||
this.apiName = apiName;
|
||||
}
|
||||
|
||||
public Long getTotalCallCount() {
|
||||
return totalCallCount;
|
||||
}
|
||||
|
||||
public void setTotalCallCount(Long totalCallCount) {
|
||||
this.totalCallCount = totalCallCount;
|
||||
}
|
||||
|
||||
public Long getUsedCallCount() {
|
||||
return usedCallCount;
|
||||
}
|
||||
|
||||
public void setUsedCallCount(Long usedCallCount) {
|
||||
this.usedCallCount = usedCallCount;
|
||||
}
|
||||
}
|
||||
@@ -27,6 +27,15 @@ public class AvgPriceQueryModel {
|
||||
Integer periodCount;
|
||||
String periodType;
|
||||
Integer houseType;
|
||||
String cat;
|
||||
|
||||
public String getCat() {
|
||||
return cat;
|
||||
}
|
||||
|
||||
public void setCat(String cat) {
|
||||
this.cat = cat;
|
||||
}
|
||||
|
||||
public Integer getPeriodCount() {
|
||||
return periodCount;
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
|
||||
/**
|
||||
* 深圳公众号成交数据录入
|
||||
*/
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\20240619--深圳成交数据.xlsx", 0, 1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年7月深圳二手房成交.xlsx", 0, 1)
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
/**
|
||||
* 将国信达及成交对标数据加入到第三方均价表,做为均价参考
|
||||
*/
|
||||
class AddThirdPriceSample {
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new AddThirdPriceSample()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
for(def i=1;i<19;i++){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\级联待优化小区集合_7月.xlsx",{d->
|
||||
if(d['A'] && d['C']){
|
||||
def dist = CommonDatautils.findDistrictById(d['A'].toString().trim())
|
||||
if(dist){
|
||||
def q = new BasicDBObject("city", dist.city).append("region", dist.region).append("name", dist.name).append("s_date", 202407)
|
||||
def udata = new BasicDBObject("zhongyuan_esf_price", d['C'].toString().toDouble().toInteger())
|
||||
|
||||
AVERAGE_STATISTICS.update(q,new BasicDBObject('$set', udata),false, false)
|
||||
}else{
|
||||
println d['A'] +" 没查到"
|
||||
}
|
||||
}
|
||||
}, i)
|
||||
|
||||
println(i)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+12
-9
@@ -34,13 +34,15 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
def fangjiaDists = new HashSet()
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataCommunityBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def rows = []
|
||||
|
||||
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
@@ -52,14 +54,18 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
def today = new Date().getDate()
|
||||
|
||||
println "starting 。。。。。。。。。"
|
||||
|
||||
def allDists = CcbCommonUtils.findAllDistrictByCity(null, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
for(def d : allDists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
for(def city : citys){
|
||||
|
||||
index ++
|
||||
|
||||
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], null)
|
||||
|
||||
println "加载建行小区量"
|
||||
|
||||
def ccbMaps = null
|
||||
if(today<=10){
|
||||
@@ -67,11 +73,11 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}else{
|
||||
ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
}
|
||||
|
||||
println "组装新数据"
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()+" 结果:"+rows.size()
|
||||
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
@@ -103,10 +109,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
|
||||
+6
-6
@@ -70,12 +70,12 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//664b73800000000000000000 5-21
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
|
||||
// println "curDate:"+curDate
|
||||
// def allCitys = CommonDatautils.findAllCity()
|
||||
// for(def city : allCitys){
|
||||
// addDealPriceMatch(city);
|
||||
// }
|
||||
// updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
println "curDate:"+curDate
|
||||
def allCitys = CommonDatautils.findAllCity()
|
||||
for(def city : allCitys){
|
||||
addDealPriceMatch(city);
|
||||
}
|
||||
updateCrawlTradeDate()//修改后抓取的成交,将时间改为本月
|
||||
|
||||
|
||||
def rows = []
|
||||
|
||||
+10
-5
@@ -20,7 +20,7 @@ import java.nio.charset.StandardCharsets
|
||||
class CcbCommonUtils {
|
||||
|
||||
|
||||
public static def citys_fjw_all = ["上海","长春","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
|
||||
public static def citys_fjw_all = ["长春","上海","扬州","北京","重庆","天津","成都","深圳","沈阳","大连","安庆","西宁","日照","武汉","西安","乌鲁木齐","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台", "徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝","阿坝州","阿拉善盟","阿里","昌都","甘南","甘孜州","果洛州","海北州","海南州","黄南州","克孜勒苏","那曲","怒江州","三沙","山南","神农架","文山州","玉树州","白沙","北屯","昌江","澄迈","定安","东方","济源","可克达拉","昆玉","乐东","临高","琼海","琼中","石河子","双河","铁门关","图木舒克","屯昌","万宁","五家渠","五指山","阿拉尔"]
|
||||
|
||||
public static def citys_all = ["乌鲁木齐","上海","长春","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水","潍坊","临沂","邯郸","济宁","威海","绵阳","梅州","平顶山","江门","赣州","泰安","襄阳","南阳","桂林","聊城","临汾","德州","商丘","衡阳","宿迁","包头","渭南","咸阳","许昌","滨州","邢台","南充","安阳","新乡","枣庄","滁州","龙岩","宝鸡","信阳","运城","衡水","东营","晋中","孝感","长治","阜阳","韶关","菏泽","德阳","宁德","荆州","乐山","驻马店","南平","汉中","榆林","宜春","鞍山","开封","周口","绥化","湘潭","黄冈","承德","焦作","牡丹江","莆田","上饶","眉山","齐齐哈尔","郴州","营口","蚌埠","大庆","赤峰","河源","吕梁","遂宁","漯河","三明","茂名","濮阳","曲靖","平凉","鄂尔多斯","十堰","吉安","黄石","佳木斯","三门峡","揭阳","忻州","通辽","泸州","六安","潮州","衢州","淮南","荆门","永州","呼伦贝尔","葫芦岛","丹东","延边","宣城","宜宾","庆阳","晋城","阳江","宿州","延安","咸宁","怀化","黄山","舟山","汕尾","盘锦","朝阳","丽水","抚州","铁岭","红河","陇南","广安","抚顺","自贡","达州","玉林","玉溪","白银","亳州","邵阳","广元","昌吉","益阳","张掖","娄底","阳泉","天水","随州","朔州","大理","通化","楚雄","巴彦淖尔","乌兰察布","新余","景德镇","松原","辽阳","阜新","酒泉","黔南","淮北","四平","梧州","黔东南","巴中","鹤壁","鸡西","内江","钦州","池州","仙桃","恩施","本溪","资阳","六盘水","百色","白城","临夏","伊犁","贵港","白山","双鸭山","吴忠","鄂州","丽江","安康","毕节","云浮","巴音郭楞","安顺","阿克苏","黑河","铜陵","雅安","石嘴山","萍乡","鹤岗","锡林郭勒盟","铜仁","防城港","喀什","兴安盟","河池","铜川","普洱","保山","凉山州","鹰潭","定西","伊春","中卫","黔西南","商洛","乌海","武威","海西州","来宾","昭通","湘西","哈密","张家界","西双版纳州","七台河","固原","攀枝花","德宏州","崇左","辽源","嘉峪关","克拉玛依","临沧","潜江","贺州","天门","海东","塔城","儋州","金昌","博尔塔拉","吐鲁番","大兴安岭","阿勒泰","和田","迪庆州","日喀则","林芝"]
|
||||
public static def citys_92 = [ "上海","扬州","北京","重庆","天津","成都","沈阳","大连","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭","陵水"]
|
||||
@@ -442,9 +442,12 @@ class CcbCommonUtils {
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("status",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
if(city){
|
||||
q.put("city", city);
|
||||
}
|
||||
|
||||
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
|
||||
// q.put("name","荣和怡景园")
|
||||
// q.put("name","一米阳光一期")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("name", new BasicDBObject('$in', ["中信泰富又一城一期","中信泰富又一城"]))
|
||||
|
||||
@@ -456,9 +459,9 @@ class CcbCommonUtils {
|
||||
}
|
||||
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
|
||||
println "[${city}] 原始:"+dbsour.count()
|
||||
def rets = []
|
||||
def rets = new HashSet()
|
||||
def ids = new HashSet()
|
||||
|
||||
def ids = []
|
||||
for(def d : dbsour){
|
||||
if(kind){
|
||||
if(d.address){
|
||||
@@ -688,6 +691,8 @@ class CcbCommonUtils {
|
||||
pgsqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static def findGuiDingIdsByCity(){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
|
||||
+109
-34
@@ -199,6 +199,16 @@ def findAllData(sqlInstance236, citycode){
|
||||
}
|
||||
|
||||
|
||||
//ExcelUtils.modify("D:\\数据\\建行\\保定202406.xlsx", { d ->
|
||||
// if(d['G']){
|
||||
// def obj = CommonDatautils.findDistrictById(d['G'])
|
||||
// if(obj){
|
||||
// d['H'] = obj.name
|
||||
// }
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
|
||||
////def url = "https://sh.anjuke.com"
|
||||
////def s = DataHandleUtils.findMatchPattern(name, "https://\\w+.anjuke.com")
|
||||
@@ -602,9 +612,9 @@ def findAllData(sqlInstance236, citycode){
|
||||
////},0)
|
||||
//
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d->
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\删除小区.xlsx",{d->
|
||||
// if(d['A']){
|
||||
// def dbo = CommonDatautils.findDistrictById(d['D'])
|
||||
// def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(dbo){
|
||||
// def udata = new BasicDBObject()
|
||||
//// udata.put("name_cat", 11)
|
||||
@@ -620,6 +630,38 @@ def findAllData(sqlInstance236, citycode){
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\添加别名.xlsx",{d->
|
||||
if(d['A'] && d['B']){
|
||||
def dbo = CommonDatautils.findDistrictById(d['A'])
|
||||
if(dbo){
|
||||
def alias = null
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}else{
|
||||
alias = dbo.alias
|
||||
}
|
||||
if(!alias.contains(d['B'])){
|
||||
alias << d['B']
|
||||
}
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("m_date",new Date())
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("_id", dbo._id)
|
||||
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+"别名已存在"
|
||||
}
|
||||
|
||||
}else{
|
||||
println dbo._id.toString() +" 没找到"
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
//def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"]
|
||||
//for(def c : citys){
|
||||
// def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city","name"])
|
||||
@@ -1289,34 +1331,34 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
|
||||
|
||||
|
||||
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240626人工指导价.xlsx"
|
||||
|
||||
ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
if(d['J']){
|
||||
// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
// if(region){
|
||||
// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240806--国信达价格录入.xlsx"
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
// if(d['J']){
|
||||
//// def region = RegionData.formatRegion(d['A'], d['J'])
|
||||
//// if(region){
|
||||
//// def dist = CommonDatautils.findDistrict(d['A'], region, d['K'], "district")
|
||||
//// if(dist){
|
||||
//// d['B'] = dist.region
|
||||
//// d['C'] = dist.name
|
||||
//// d['L'] = dist.status-dist.name_cat
|
||||
//// }
|
||||
//// }
|
||||
//// if(d['J'] instanceof String){
|
||||
// def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
// if(dist){
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
// d['C'] = dist.name
|
||||
// d['L'] = dist.status-dist.name_cat
|
||||
// }
|
||||
// }
|
||||
// if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
// d['K'] = dist.estate_type2
|
||||
// d['L'] = dist.status
|
||||
// d['M'] = dist.name_cat
|
||||
|
||||
// d['H'] = dist.name
|
||||
}
|
||||
// }
|
||||
}
|
||||
},0)
|
||||
//
|
||||
//// d['H'] = dist.name
|
||||
// }
|
||||
//// }
|
||||
// }
|
||||
//},0)
|
||||
|
||||
|
||||
//def file = "D:\\数据\\建行\\房产估值\\指导价\\5城价格翻新6-21.xlsx"
|
||||
@@ -1352,24 +1394,57 @@ ExcelUtils.multiThreadingModify(fileName, {d->
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
//def filePath = "D:\\数据\\建行\\房产估值\\指导价\\大庆删除小区.xlsx"
|
||||
//
|
||||
//def filePath = "D:\\数据\\数据建设\\小区\\嘉峪关处理.xlsx"
|
||||
//ExcelUtils.modify(filePath, { d ->
|
||||
// if(d['A']){
|
||||
//
|
||||
// def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
// if(obj){
|
||||
//
|
||||
// def data = new BasicDBObject()
|
||||
// data.put("status", 1)
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(d['A'])), new BasicDBObject('$set', data))
|
||||
//// def obj = CommonDatautils.findDistrictById(d['A'])
|
||||
// def obj = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// def list = CommonDatautils.findDistrictList(d['A'], null, d['C'], "district")
|
||||
// if(list){
|
||||
// if(list.size()==1){
|
||||
// def dbo = list[0]
|
||||
// if(dbo){
|
||||
// def data = new BasicDBObject()
|
||||
// data.put("region", d['B'])
|
||||
// data.put("m_date", new Date())
|
||||
// DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', data))
|
||||
// }
|
||||
// }else{
|
||||
// println d['C'] +" 有多个"
|
||||
// }
|
||||
// }
|
||||
//// if(obj){
|
||||
//// def lng = d['G']
|
||||
//// def lat =d['H']
|
||||
////
|
||||
//// // 高德转百度
|
||||
//// def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3)
|
||||
//// def lng2 = baidu.x
|
||||
//// def lat2 = baidu.y
|
||||
////
|
||||
////
|
||||
//// def data = new BasicDBObject()
|
||||
//// Long lng_new = d['G'].toString().toDouble() *10000000000L
|
||||
//// Long lat_new = d['H'].toString().toDouble() *10000000000L
|
||||
//// data.put("lng", lng_new)
|
||||
//// data.put("lat", lat_new)
|
||||
////
|
||||
//// Long dLat2 = lat2 * 10000000000L
|
||||
//// Long dLng2 = lng2 * 10000000000L
|
||||
//// data.put("lat2", dLat2)
|
||||
//// data.put("lng2", dLng2)
|
||||
////
|
||||
////// data.put("status", 1)
|
||||
//// data.put("m_date", new Date())
|
||||
//// DbUtils.seaweed.update(new BasicDBObject('_id',obj._id), new BasicDBObject('$set', data))
|
||||
//// }
|
||||
// }
|
||||
//
|
||||
//})
|
||||
|
||||
|
||||
|
||||
//def fileName = "D:\\数据\\建行\\房产估值\\指导价\\2024-04-08修改人工指导价.xlsx"
|
||||
//ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
// if(d['B'] && !d['D']){
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
ExcelUtils.modify("D:\\数据\\建行\\东莞202406.xlsx", { d ->
|
||||
if(d['G'] && d['G'] instanceof String){
|
||||
|
||||
def obj = CommonDatautils.findDistrictById(d['G'])
|
||||
if(obj){
|
||||
d['H'] = obj.name
|
||||
}
|
||||
}
|
||||
})
|
||||
+9
@@ -257,6 +257,9 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -303,6 +306,8 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "·¿¶à¶à"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "ÎÒ°®ÎÒ¼Ò"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "·¿²ú³¬ÊÐ"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -584,6 +589,10 @@ class ThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
|
||||
+67
-3
@@ -32,6 +32,10 @@ class TradeStatLog {
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pssqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
@@ -41,7 +45,9 @@ class TradeStatLog {
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
// task.exportTrade()//导出未入库成交
|
||||
|
||||
task.exportTrade2()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -584,6 +590,36 @@ class TradeStatLog {
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据")
|
||||
}
|
||||
|
||||
|
||||
def exportTrade2(){
|
||||
|
||||
// def startDate = "2024-06-01"
|
||||
// def endDate = "2024-07-01"
|
||||
def ids = findAllDealIds();
|
||||
println ids.size()
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\2024年6月成交数据.xlsx",{d->
|
||||
if(ids.contains(d['A'])){
|
||||
d['V'] = "供"
|
||||
}
|
||||
if(d['S'] && !d['W']){
|
||||
def dist = CommonDatautils.findDistrictById(d['S'])
|
||||
if(!dist.name_cat && dist.name_cat!=0){
|
||||
d['W'] = "name_cat过滤"
|
||||
}
|
||||
if(dist.name_cat != 0 || !dist.lng || !["普通住宅","商住"].contains(dist.estate_type2) || dist.name.contains("洋房") || dist.name.contains("公房")){
|
||||
d['W'] = "小区过滤"
|
||||
}
|
||||
def key = d['B']+"##"+d['Q']+"##"+d['R']+"##"+d['D']+"##"+d['E'].toString().toDouble().toInteger()+"##"+d['J'].toString().substring(0,10)
|
||||
def map = findDealByCommId(d['B'], d['S'])
|
||||
def count = map.get(key)
|
||||
if(count>=2){
|
||||
d['W'] = "重复"
|
||||
}
|
||||
}
|
||||
println d['A']
|
||||
})
|
||||
}
|
||||
|
||||
def deleteDistrict(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\要删除小区.xlsx",{d->
|
||||
if(d['G']){
|
||||
@@ -637,11 +673,39 @@ class TradeStatLog {
|
||||
return trades
|
||||
}
|
||||
|
||||
def findAllDealIds(){
|
||||
def sql = "select gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where p9_start_date='2024-07-28'"
|
||||
def ids = []
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.gujia_bds_mdl_id)){
|
||||
ids << r.gujia_bds_mdl_id
|
||||
}
|
||||
})
|
||||
return ids
|
||||
}
|
||||
|
||||
def findDealByCommId(city, commid){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_id", commid)
|
||||
q.put("city", city)
|
||||
q.put('trade_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2024-06-01")).append('$lt', DateUtils.parseFromTightYYYYMMDD("2024-07-01")));
|
||||
def dbcursor = DEAL_PRICE_COLL.find(q)
|
||||
def map = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
|
||||
def count = map.get(key)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
map.put(key, count+1)
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
def findDealIds(city, startDate, endDate){
|
||||
def sql = "select * from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate<to_date('${endDate}','yyyy-mm-dd') and cityname='${city}'";//and dealdate<to_date('2021-08-01','yyyy-mm-dd')
|
||||
def ids = []
|
||||
def detailInfos = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
pssqlInstance.eachRow(sql,{r->
|
||||
if(!ids.contains(r.dealid)){
|
||||
ids << r.dealid
|
||||
|
||||
|
||||
+38
-38
@@ -146,7 +146,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataCommunityGJRH(){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -221,7 +221,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -253,7 +253,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// FJW_000000_000000_EVALUATE_RATE_CITY_FLW_ADD_
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_CITY_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -295,7 +295,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
// FJW_000000_000000_EVALUATE_RATE_COMM_FLW_ADD_ avm_evaluate_rate_comm_gj_h
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_COMM_FJW_HIS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -327,7 +327,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -356,7 +356,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -384,7 +384,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -422,7 +422,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
});
|
||||
}
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -452,7 +452,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -485,7 +485,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -517,7 +517,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -552,7 +552,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -576,7 +576,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
index++
|
||||
def curStartStr = curStartDateStr
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
|
||||
}
|
||||
@@ -676,7 +676,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -716,7 +716,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
|
||||
|
||||
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -987,7 +987,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -998,7 +998,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataBuildingFJW(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1072,7 +1072,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
allRows << data
|
||||
});
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1084,7 +1084,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportThirdPartyPrice2(price_date, new_dists) {
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1132,7 +1132,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgThirdPartyPrice2(price_date, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1215,7 +1215,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataBuildprojFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1261,7 +1261,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1304,7 +1304,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
}
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1319,7 +1319,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1393,7 +1393,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgCommunityPdescribeGj(month){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMM_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1460,7 +1460,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportGuideStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1501,7 +1501,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgGuideStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1537,7 +1537,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgBigdataSupplierStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1570,7 +1570,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataSupplierStatistics(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1609,7 +1609,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportBigdataModelingBase(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1652,7 +1652,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
def exportPgBigdataModelingBase(time){
|
||||
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1689,7 +1689,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCityPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1737,7 +1737,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataDistPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_DISTPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1784,7 +1784,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCommPrice(queryDate){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_ronghe\\${curDate}\\FJW_000000_000000_BIGDATA_COMMPRICE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1845,7 +1845,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataCommlayoutpicFjw(){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1856,7 +1856,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportBigdataForsale(publishDate, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -1913,7 +1913,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportPgBigdataForsale(publishDate,endDate, new_dists){
|
||||
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
|
||||
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
def fileName = "/home/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
|
||||
}
|
||||
@@ -2098,7 +2098,7 @@ class ExportDistrictRongHe extends BaseBatch{
|
||||
|
||||
def exportOk(curDate){
|
||||
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
|
||||
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
|
||||
def dataCodeFileName = "/home/ccb_rh/${curDate}/${curDate}.ok"
|
||||
if(CommonDatautils.isWindow()){
|
||||
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
|
||||
}
|
||||
|
||||
+67
-3
@@ -1209,6 +1209,13 @@ class CommonDatautils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
public static def findAllCount(String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("cat", cat);
|
||||
q.put("status", 0)
|
||||
return SEAWEED_MONGO_COLL.count(q)
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrictsByName(String city, String name, String cat){
|
||||
|
||||
@@ -4653,6 +4660,7 @@ class CommonDatautils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","区域")
|
||||
q.put("district_diff",1)
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4662,7 +4670,7 @@ class CommonDatautils {
|
||||
keys.put("mzb_district_name", 1)
|
||||
keys.put("shape", 1)
|
||||
keys.put("district_short",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(2000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def regionMap = [:]
|
||||
for(def d : dbcursor){
|
||||
if(d.mzb_district_name){
|
||||
@@ -4686,7 +4694,7 @@ class CommonDatautils {
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
// q.put("province",new BasicDBObject('$in', ['浙江省','江苏省','山东省']))
|
||||
// q.put("city","芜湖市")
|
||||
q.put("city",new BasicDBObject('$in', ["中山市","东莞市","东方市"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
@@ -4712,10 +4720,42 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
|
||||
static def queryTownByCity(city){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",city)
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("province",1)
|
||||
keys.put("city",1)
|
||||
keys.put("district",1)
|
||||
keys.put("street",1)
|
||||
keys.put("shape",1)
|
||||
keys.put("alias",1)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q, keys).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def townMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def key = d.district
|
||||
def list = townMap.get(key)
|
||||
if(!list){
|
||||
list = []
|
||||
}
|
||||
list << d
|
||||
townMap.put(key, list)
|
||||
}
|
||||
|
||||
println "乡镇街道 加载完成"
|
||||
return townMap
|
||||
}
|
||||
|
||||
|
||||
static def queryNewTownByCity(){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("type","乡镇街道")
|
||||
q.put("city",new BasicDBObject('$in', ["中山","东莞","东方"]))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
@@ -4833,6 +4873,9 @@ class CommonDatautils {
|
||||
def jsonObject = null
|
||||
def stop = false
|
||||
def count = 0
|
||||
def road = ""
|
||||
def name = ""
|
||||
def type = ""
|
||||
while(!stop){
|
||||
if(glocation){
|
||||
jsonObject = GaoDeApiUtils.geocode(glocation)
|
||||
@@ -4852,8 +4895,26 @@ class CommonDatautils {
|
||||
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
|
||||
if(addressComponent.streetNumber){
|
||||
if(addressComponent.streetNumber.street){
|
||||
road = addressComponent.streetNumber.street
|
||||
}
|
||||
if(addressComponent.streetNumber.number){
|
||||
road = road + addressComponent.streetNumber.number
|
||||
}
|
||||
}
|
||||
|
||||
if(addressComponent.building){
|
||||
if(addressComponent.building.name){
|
||||
name = addressComponent.building.name
|
||||
}
|
||||
if(addressComponent.building.type){
|
||||
type = addressComponent.building.type
|
||||
}
|
||||
}
|
||||
|
||||
format_addr = jsonObject.regeocode.formatted_address
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4871,6 +4932,9 @@ class CommonDatautils {
|
||||
map.district = district
|
||||
map.township = township
|
||||
map.format_addr = format_addr
|
||||
map.road = road
|
||||
map.type = type
|
||||
map.name = name
|
||||
return map;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class GreenBuild {
|
||||
|
||||
DBCollection GREEN_BUILD_COLL = MongoUtils.getCollectionByName("crawl", "green_build_project", "192.168.0.206",27020)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new GreenBuild()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAll()
|
||||
def rets = match(datas)
|
||||
ExcelUtils.write("D:\\数据\\绿色建筑\\绿色建筑匹配住宅.xlsx", rets, "匹配结果")
|
||||
}
|
||||
|
||||
def match(datas){
|
||||
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
|
||||
rets << ["ID", "城市","区域", "名称", "匹配ID", "匹配城市" , "匹配区域" ,"匹配名称"]
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def data = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def jsonObject = FangjiaClientUtils.newMatchPrice(data.city, data.name, "property", false)
|
||||
if(jsonObject && jsonObject.containsKey("result")) {
|
||||
def resultMap = jsonObject.get("result")
|
||||
data.match_id = resultMap.get("id").toString()
|
||||
data.match_city = resultMap.get("city")
|
||||
data.match_district = resultMap.get("district")
|
||||
data.match_name = resultMap.get("name")
|
||||
rets << [data._id.toString(), data.city, data.district, data.name, data.match_id, data.match_city, data.match_district, data.match_name]
|
||||
}else{
|
||||
rets << [data._id.toString(), data.city, data.district, data.name]
|
||||
}
|
||||
|
||||
println totalCount.get()
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
println "查询完成......."
|
||||
latch.await();
|
||||
return rets
|
||||
}
|
||||
|
||||
def findAll(){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("name", 1)
|
||||
return GREEN_BUILD_COLL.find(new BasicDBObject(), keys).toList()
|
||||
}
|
||||
|
||||
}
|
||||
@@ -12,8 +12,8 @@ class OpenUserBatch {
|
||||
def task = new OpenUserBatch();
|
||||
// task.run();
|
||||
// task.test();
|
||||
// task.addShenZhenFjtUser();
|
||||
task.addFujianFjtUser()
|
||||
task.addShenZhenFjtUser();
|
||||
// task.addFujianFjtUser()
|
||||
}
|
||||
|
||||
def test(){
|
||||
@@ -46,20 +46,20 @@ class OpenUserBatch {
|
||||
}
|
||||
|
||||
def addShenZhenFjtUser(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\渤海\\开通线上评估客户经理名单.xlsx",0,1)
|
||||
def datas = ExcelUtils.read("D:\\数据\\黑龙江银行\\大正银行端新增账户模板中国银行.xls",0,1)
|
||||
def i = 0;
|
||||
for(def data : datas){
|
||||
i++;
|
||||
def id = 252608 + i;
|
||||
def id = 252893 + i;
|
||||
BigDecimal bd = new BigDecimal(data[3].toString());
|
||||
String mobile = bd.toPlainString();
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '17', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2023-01-01 00:00:00', '0', '', NULL)";
|
||||
def sql = "insert into users(id, email, password, role, user_role, city, gold_total, come_from, date_created, last_updated, status, uid, access_token, house_report_token, open_day, fangdiao_apply_add, gujia_add, fangdiao_apply_count, gujia_down_count, fangdiao_add, fangdiao_down_count, yang_ben_down, mobile, pay_password, evaluation_count, close_date, manager, role_users, weixinId) VALUES ("+id+", '"+mobile+"', '666666', '18', NULL, NULL, '0', 'fangjiatong', now(), now(), '0', NULL, NULL, '', '0', '2', '2', '2', '500', NULL, NULL, '0', NULL, NULL, NULL, '2025-12-31 00:00:00', '0', '', NULL)";
|
||||
onLineSqlInstance.execute(sql);
|
||||
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '姚静梦')";
|
||||
def sql2 = "INSERT INTO users_detail (id, p_id,name, org_name, phone, fax, email, address, role, company, user_valid_date, online_status, last_login_date, last_operat_date, date_created, last_updated, mobile, eva_guide, ip_list, sale_name) VALUES ('"+id+"', '0', '"+data[2]+"', '"+data[1]+"', '"+mobile+"', NULL, NULL, NULL, NULL, NULL, NULL, b'0', now(), now(), now(), now(), NULL, b'0', '', '王蕾')";
|
||||
onLineSqlInstance.execute(sql2)
|
||||
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'上海','上海:')"
|
||||
def sql3 = "INSERT INTO user_city (user_id, citys, credits) VALUES("+id+",'哈尔滨,齐齐哈尔,大庆,牡丹江,佳木斯,绥化,鸡西,七台河,双鸭山,鹤岗,黑河,伊春,大兴安岭','哈尔滨:,齐齐哈尔:,大庆:,牡丹江:,佳木斯:,绥化:,鸡西:,七台河:,双鸭山:,鹤岗:,黑河:,伊春:,大兴安岭:')"
|
||||
onLineSqlInstance.execute(sql3)
|
||||
}
|
||||
}
|
||||
|
||||
+247
File diff suppressed because one or more lines are too long
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.google.gson.JsonObject
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
class BaiduBaikeTown {
|
||||
|
||||
DBCollection dzb_COLL = MongoUtils.getCollectionByName("crawl", "development_zone_bdbk", "192.168.0.206",27020)
|
||||
|
||||
|
||||
static main(args) {
|
||||
def task = new BaiduBaikeTown()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",{row->
|
||||
if(!row['E'] && row['A']=="郑州经济技术开发区"){
|
||||
def q = new BasicDBObject()
|
||||
q.put("desc",new BasicDBObject('$ne', 'null'))
|
||||
q.put("name",row['A'])
|
||||
def d = dzb_COLL.findOne(q)
|
||||
def townMap = parse(d.desc, d.name)
|
||||
row['D'] = townMap.town_str1
|
||||
row['F'] = townMap.town_str2
|
||||
|
||||
// def ret = []
|
||||
// ret << ["名称", "基本信息", "目录", "全文"]
|
||||
// for(def d : dbcursor){
|
||||
// def townMap = parse(d.desc, d.name)
|
||||
// if(townMap.town){
|
||||
// println d.name+" "+townMap.town
|
||||
// }
|
||||
// ret << [d.name, townMap.town, townMap.towns, townMap.town_str1, townMap.town_str2]
|
||||
// }
|
||||
}
|
||||
|
||||
})
|
||||
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx", ret, "数据")
|
||||
}
|
||||
|
||||
def parse(desc, name){
|
||||
def map = [:]
|
||||
if(desc){
|
||||
def doc = Jsoup.parse(desc)
|
||||
def town = JQueryUtils.selectText(".basicInfoBlock_Ex6IU div:contains(下辖地区) dd", doc)
|
||||
|
||||
def s = doc.toString()
|
||||
def dataStr = DataHandleUtils.findMatchPattern(s,"PAGE_DATA=.*</script>").replace("PAGE_DATA= ","").replace("</script>","")
|
||||
def jsonObj = JSONObject.fromObject(dataStr)
|
||||
|
||||
def str = DataHandleUtils.findMatchPattern(doc.text(),"下辖.*[街道|镇]、?和?")
|
||||
def towns = []
|
||||
if(jsonObj && jsonObj.catalog){
|
||||
def catalogs = jsonObj.catalog
|
||||
def index = null
|
||||
for(def log : catalogs){
|
||||
def curIndex = log.index
|
||||
if(log.title=="行政区划"){
|
||||
index = log.index
|
||||
}
|
||||
if(index && curIndex.toString().startsWith(index+"-")){
|
||||
towns << log.title
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
map.town_str1 = str
|
||||
if(town){
|
||||
map.town = town
|
||||
}else{
|
||||
map.town = ""
|
||||
}
|
||||
|
||||
if(towns){
|
||||
map.towns = towns.join(",")
|
||||
}else{
|
||||
map.towns = ""
|
||||
}
|
||||
|
||||
|
||||
def str2 = DataHandleUtils.findMatchPattern(doc.text(),"开发区辖.*[,|。]")
|
||||
map.town_str2 = str2
|
||||
return map
|
||||
}
|
||||
map.town_str1 = ""
|
||||
map.town_str2 = ""
|
||||
map.town = ""
|
||||
map.towns = ""
|
||||
return map
|
||||
}
|
||||
}
|
||||
+221
-11
@@ -7,6 +7,7 @@ import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.AddressUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
@@ -17,17 +18,23 @@ import net.sf.json.JSONObject
|
||||
import org.apache.poi.ss.util.RegionUtil
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DevProcess {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection DEV_MAPPING_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
DBCollection DEV_MAPPING_CONN2 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping_2", MONGO100_URL)
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
// DBCollection kfq_COLL = MongoUtils.getCollectionByName("crawl", "kfq_investchn_com", "192.168.0.206",27020)
|
||||
|
||||
static main(args) {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
@@ -41,6 +48,7 @@ class DevProcess {
|
||||
// task.updateCityAndDist()
|
||||
// task.addDevAddres1() //按添加地址中包含开发区名称的地址
|
||||
// task.addDevAddres2()//添加相差2公里范围内的地址
|
||||
// task.addDevAddres3()
|
||||
// task.matchDevAndAddAddress()
|
||||
|
||||
// task.stat()
|
||||
@@ -48,7 +56,7 @@ class DevProcess {
|
||||
// task.createDevMapping()
|
||||
// task.deleteAddr()
|
||||
// task.deleteAddr2()
|
||||
// task.synchDevTo43()
|
||||
// task.synchDevTo100()
|
||||
// task.addNewDev2()
|
||||
|
||||
// task.addAlias()
|
||||
@@ -56,6 +64,17 @@ class DevProcess {
|
||||
// task.matchDevTownship()//添加乡镇街道
|
||||
|
||||
// task.addDevStreet()//添加区域为等级开发区下的街道到开发区对应表中
|
||||
|
||||
// task.matchNewCrawlDev()
|
||||
|
||||
// println DateUtils.add("2024-7-24", 7);
|
||||
|
||||
// task.addKfc()
|
||||
// task.devToPark() //开发区下辖园区
|
||||
// task.checkTown()
|
||||
|
||||
// task.repeatSave()
|
||||
|
||||
}
|
||||
|
||||
|
||||
@@ -334,15 +353,27 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\福建省.xlsx",{d->
|
||||
if(!d['Q'] && d['L']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
|
||||
if(!d['I'] && d['J']){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['L'])
|
||||
q.put("city",d['G'])
|
||||
// q.put("name", d['J'])
|
||||
// q.put("city",d['G'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['J']]))
|
||||
def name = d['J'].replace('经济技术开发区','经济开发区')
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['Q'] = obj._id.toString()
|
||||
d['R'] = obj.name
|
||||
d['I'] = "库中有开发区-相似2"
|
||||
d['K'] = obj.name
|
||||
// d['C'] = obj._id.toString()
|
||||
// d['D'] = obj.name
|
||||
// if(obj.name == d['A']){
|
||||
// d['E'] = "相同"
|
||||
// }else{
|
||||
// d['E'] = "包含"
|
||||
// }
|
||||
// d['E'] = "别名"
|
||||
// d['F'] = obj.name
|
||||
}
|
||||
}
|
||||
},0);
|
||||
@@ -509,6 +540,38 @@ class DevProcess {
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
|
||||
def addDevAddres3(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\新跑省份\\新疆省入库.xlsx",{d->
|
||||
if(d['A']){
|
||||
def obj = development_zone_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
d['C'] = obj._id.toString()
|
||||
|
||||
def q2 = new BasicDBObject()
|
||||
q2.put("name", d['A'])
|
||||
q2.put("address", d['B'])
|
||||
q2.put("type","地址")
|
||||
def sameObj = DEV_MAPPING_CONN.findOne(q2)
|
||||
if(!sameObj){
|
||||
index++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province", obj.province)
|
||||
udata.put("city", obj.city)
|
||||
udata.put("district", obj.district)
|
||||
udata.put("name",obj.name)
|
||||
udata.put("type", "地址")
|
||||
udata.put("address", d['B'])
|
||||
DEV_MAPPING_CONN.save(udata)
|
||||
}
|
||||
}
|
||||
}
|
||||
},0)
|
||||
println index
|
||||
// ExcelUtils.write("D:\\数据\\数据建设\\待新增开发区.xlsx",rets, "新增")
|
||||
}
|
||||
|
||||
def stat(){
|
||||
def dbcursor = development_zone_CONN.find(new BasicDBObject())
|
||||
def total = 0
|
||||
@@ -633,13 +696,13 @@ class DevProcess {
|
||||
});
|
||||
}
|
||||
|
||||
def synchDevTo43(){
|
||||
def dbcursor = development_zone_CONN.find()
|
||||
def synchDevTo100(){
|
||||
def dbcursor = DEVELOPMENT_ZONE_43_COLL.find()
|
||||
def rets = []
|
||||
for(def r : dbcursor){
|
||||
rets << r
|
||||
}
|
||||
DEVELOPMENT_ZONE_43_COLL.insert(rets)
|
||||
development_zone_CONN.insert(rets)
|
||||
}
|
||||
|
||||
|
||||
@@ -777,4 +840,151 @@ class DevProcess {
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
def matchNewCrawlDev(){
|
||||
def query = new BasicDBObject()
|
||||
query.put("match_id", new BasicDBObject('$exists', false))
|
||||
def list = kfq_COLL.find(query).toList()
|
||||
for(def d : list){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("name", d.name)
|
||||
q.put('alias', new BasicDBObject('$in',[d.name]))
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_id", obj._id.toString())
|
||||
udata.put("match_name", obj.name)
|
||||
kfq_COLL.update(new BasicDBObject("_id",d._id), new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addKfc(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\投促中国开发区(1).xlsx",{d->
|
||||
if(d['I']){
|
||||
if(d['I'] == "添加"){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("create_time" , new Date())
|
||||
udata.put("industry",d['D'])
|
||||
if(d['E'].contains("国家级")){
|
||||
udata.put("level" ,"国家级")
|
||||
}else if(d['E'].contains("省级")){
|
||||
udata.put("level" ,"省级")
|
||||
}
|
||||
udata.put("name" ,d['F'])
|
||||
udata.put("province" ,d['G'])
|
||||
udata.put("type" ,d['H'])
|
||||
udata.put("city" ,d['B'])
|
||||
udata.put("district" ,d['J'])
|
||||
development_zone_CONN.save(udata)
|
||||
}else{
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", d['I'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
def alias = []
|
||||
if(obj.alias){
|
||||
alias = obj.alias
|
||||
}
|
||||
if(!alias.contains(d['F'])){
|
||||
alias << d['F']
|
||||
}
|
||||
if(alias && alias.size()!=obj.alias.size()){
|
||||
def updateData = new BasicDBObject()
|
||||
updateData.put("alias", alias)
|
||||
development_zone_CONN.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',updateData))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def devToPark(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\数据建设\\开发区\\百度百科下辖街道解析.xlsx",0, 1)
|
||||
def rets = []
|
||||
rets << ["开发区名称", "下辖街道", "城市", "区域"]
|
||||
for(def d : datas){
|
||||
if(d['E']){
|
||||
def arrs = d['E'].split("、")
|
||||
def q = new BasicDBObject().append("name", d['A'])
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
for(def park : arrs){
|
||||
rets << [d['A'], park, obj.city, obj.district]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx", rets, "下辖街道")
|
||||
}
|
||||
|
||||
|
||||
def checkTown(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\百度百科开发区下辖街道.xlsx",{d->
|
||||
|
||||
if(!d['E']){
|
||||
if(d['A']){
|
||||
// def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
|
||||
// if(jsonObject1) {
|
||||
// if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
|
||||
// def township = jsonObject1.get("township")
|
||||
// def district = jsonObject1.get("district")
|
||||
// d['E'] = "接口"
|
||||
// if(district && district!='null'){
|
||||
// d['F'] = district
|
||||
// }
|
||||
// if(township && township!='null'){
|
||||
// d['G'] = township
|
||||
// }
|
||||
//
|
||||
// d['H'] = jsonObject1.get("townCode")
|
||||
// if(jsonObject1.get("provinceCode") && jsonObject1.get("provinceCode")!='null'){
|
||||
// d['I'] = jsonObject1.get("provinceCode")
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['C'])
|
||||
// q.put("district", d['D'])
|
||||
// q.put("mzb_district_name", d['D'])
|
||||
q.put("type", "乡镇街道")
|
||||
q.put('street',new BasicDBObject('$regex', Pattern.compile(d['B'].toString().trim()+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
// q.put("street", d['B'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['B']]))
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
d['E'] = "可入库-忽略区域模糊2"
|
||||
d['F'] = obj.district
|
||||
d['G'] = obj.street
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
def repeatSave(){
|
||||
def dbcursor = DEV_MAPPING_CONN.find(new BasicDBObject())
|
||||
def list = []
|
||||
for(def map : dbcursor){
|
||||
def columns = ["_id","province","city","district","name","type","address","dev_district", "township","park"]
|
||||
def udata = new BasicDBObject()
|
||||
map.each{k,v->
|
||||
if(columns.contains(k)){
|
||||
if(v instanceof String){
|
||||
if(k=="_id"){
|
||||
udata.put("_id", new ObjectId(v))
|
||||
}else{
|
||||
udata.put(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
list << udata
|
||||
}
|
||||
DEV_MAPPING_CONN2.insert(list)
|
||||
}
|
||||
}
|
||||
|
||||
+281
@@ -0,0 +1,281 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.AMQP
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.support.WriteRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.xcontent.XContentType
|
||||
|
||||
class DevUpdateTestBatch {
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_100_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
DBCollection DEV_MAPPING_100_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
def task = new DevUpdateTestBatch()
|
||||
def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx
|
||||
// task.addDev(path)
|
||||
// task.addDevAddress(path)
|
||||
|
||||
// task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是")
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
def addDev(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['I'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(!obj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", d['B'])
|
||||
o.put("city", d['C'])
|
||||
o.put("district", d['D'])
|
||||
o.put("name", d['A'])
|
||||
o.put("level", d['E'])
|
||||
o.put("type", d['F'])
|
||||
|
||||
if(d['G']){
|
||||
o.put("township", d['G'])
|
||||
}
|
||||
if(d['H']){
|
||||
def arr = d['H'].split(",")
|
||||
if(arr){
|
||||
o.put("alias", arr)
|
||||
}
|
||||
}
|
||||
def id = addMongoDB(o)
|
||||
o.remove("_id")
|
||||
o.id = id.toString()
|
||||
o.remove("create_time")
|
||||
addEs(o, "192.168.0.89", "development_zone")
|
||||
addEs(o, "192.168.0.52", "development_zone")
|
||||
d['I'] = "已完成"
|
||||
}else{
|
||||
println d['A'] +" 已存在"
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addMongoDB(o){
|
||||
o.put("create_time", new Date())
|
||||
development_zone_100_CONN.save(o)
|
||||
DEVELOPMENT_ZONE_43_COLL.save(o)
|
||||
return o._id
|
||||
}
|
||||
|
||||
def addEs(map, host, indexName){
|
||||
BulkRequest request = new BulkRequest();
|
||||
int processedCount = 0;
|
||||
def client = null
|
||||
if(host && host == "192.168.0.89"){
|
||||
client = ElasticUtils89.getClient(host)
|
||||
}else if(host && host == "192.168.0.52"){
|
||||
client = ElasticUtils.getClient(host)
|
||||
}
|
||||
|
||||
request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON));
|
||||
if (request.numberOfActions() > 0) {
|
||||
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||
if (bulkResponse.hasFailures()) {
|
||||
// 处理错误
|
||||
println "Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||
} else {
|
||||
processedCount += request.numberOfActions();
|
||||
println "Processed " + processedCount + " documents"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def addDevAddress(path){
|
||||
ExcelUtils.modify(path,{ d->
|
||||
if(!d['D'] && d['A']){
|
||||
def obj = development_zone_100_CONN.findOne(new BasicDBObject("name", d['A']))
|
||||
if(obj){
|
||||
def q = new BasicDBObject("name", d['A']).append("type",d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
q.put("township", d['C'])
|
||||
}else if(d['B'] == "地址"){
|
||||
q.put("address", d['C'])
|
||||
}else if(d['B'] == "园区"){
|
||||
q.put("park", d['C'])
|
||||
}
|
||||
def mappingObj = DEV_MAPPING_100_CONN.findOne(q)
|
||||
if(!mappingObj){
|
||||
def o = new BasicDBObject()
|
||||
o.put("province", obj.province)
|
||||
o.put("city", obj.city)
|
||||
o.put("district", obj.district)
|
||||
o.put("name", obj.name)
|
||||
o.put("type", d['B'])
|
||||
if(d['B'] == "乡镇街道"){
|
||||
o.put("township", d['C'])
|
||||
}else if(d['B'] == "地址"){
|
||||
o.put("address", d['C'])
|
||||
}else if(d['B'] == "园区"){
|
||||
o.put("park", d['C'])
|
||||
}
|
||||
def date = new Date()
|
||||
o.put("c_date", date)
|
||||
o.put("version", date.getTime())
|
||||
DEV_MAPPING_100_CONN.save(o)
|
||||
|
||||
|
||||
// o.id = o._id.toString()
|
||||
// o.remove("_id")
|
||||
// o.remove("c_date")
|
||||
// addEs(o, "192.168.0.89", "dev_mapping")
|
||||
// try{
|
||||
// addEs(o, "192.168.0.52", "dev_mapping")
|
||||
// }catch(Exception e){
|
||||
// println "52 es error: "+e.printStackTrace()
|
||||
// }
|
||||
|
||||
d['D'] = "已完成"
|
||||
}else{
|
||||
d['D'] = "重复"
|
||||
println d['A'] +" "+ d['C']+" 重复"
|
||||
}
|
||||
}else{
|
||||
d['D'] = "不存在"
|
||||
println d['A'] +" 不存在"
|
||||
}
|
||||
}
|
||||
},1)
|
||||
}
|
||||
|
||||
|
||||
def updateHasGov(province, city, district, street, hasGov){
|
||||
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", province)
|
||||
q.put("city", city)
|
||||
q.put("district", district)
|
||||
q.put("street", street)
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
//修改43
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',new BasicDBObject('has_gov',hasGov)))
|
||||
|
||||
//修改89 administrative_mapping
|
||||
update89EsAdministrativeMapping(province, city, district, street, hasGov)
|
||||
//修改89 administrative_mapping
|
||||
update52EsAdministrativeMapping(province, city, district, street, hasGov)
|
||||
}else{
|
||||
println province+" "+city+" "+district+" "+street+" 没找到"
|
||||
}
|
||||
//修改100 community
|
||||
//修改89 community
|
||||
//修改52 community
|
||||
|
||||
}
|
||||
|
||||
|
||||
def update89EsAdministrativeMapping(province, city, district, street, hasGov){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.has_gov='${hasGov}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"province_name": "${province}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"city_name":"${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"area_name":"${district}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"street_name":"${street}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
println(res)
|
||||
}
|
||||
|
||||
|
||||
def update52EsAdministrativeMapping(province, city, district, street, hasGov){
|
||||
def header = [:]
|
||||
header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString()
|
||||
String seawaterUrl = "http://192.168.0.52:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false"
|
||||
def body = """
|
||||
{
|
||||
"script": {
|
||||
"source": "ctx._source.has_gov='${hasGov}'",
|
||||
"lang": "painless"
|
||||
},
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"match": {
|
||||
"province_name": "${province}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"city_name":"${city}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"area_name":"${district}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"match": {
|
||||
"street_name":"${street}"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
HttpRequest request = new HttpRequest()
|
||||
String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8")
|
||||
// def response = JSONObject.fromObject(res)
|
||||
println(res)
|
||||
}
|
||||
}
|
||||
+34
-1
@@ -10,7 +10,8 @@ class DistrictAliasProcess {
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictAliasProcess();
|
||||
task.run()
|
||||
// task.run()
|
||||
task.fenQiProcess()
|
||||
}
|
||||
|
||||
def run(){
|
||||
@@ -42,4 +43,36 @@ class DistrictAliasProcess {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def fenQiProcess(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("cat","district")
|
||||
q.put("status", 0)
|
||||
q.put("master_district_name", 'master')
|
||||
def dbcursor = DbUtils.seaweed.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << ["_id":d._id.toString(),"city":d.city, "region":d.region, "name":d.name]
|
||||
}
|
||||
|
||||
for(def r : rets){
|
||||
def mainName1 = r.name.toString().replaceAll("([一二三四五六七八九十百千\\d]+)期\$", "")
|
||||
def mainName2 = r.name.toString().replaceAll("(\\w|东|南|北|中|西|甲|乙|丙|丁|一|二|三|四|五|六|七|八|九|十)区\$", "")
|
||||
def lastObj = null
|
||||
for(def j : rets){
|
||||
if((j.name==mainName1 || mainName2==j.name) && r._id!=j._id && r.region==j.region && r.city==j.city){
|
||||
lastObj = j;
|
||||
break
|
||||
}
|
||||
}
|
||||
if(lastObj){
|
||||
println r
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("master_district_name","")
|
||||
udata.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", new ObjectId(r._id)), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+5
-5
@@ -28,23 +28,23 @@ class GXDdistrictMatch {
|
||||
static main(args) {
|
||||
def task = new GXDdistrictMatch();
|
||||
// task.run()
|
||||
// task.addOldMatch()
|
||||
task.addOldMatch()
|
||||
// task.runDb()
|
||||
|
||||
// task.updateDistStatus()//恢复name_cat不正常数据
|
||||
// task.saveToDB() //保存新增小区
|
||||
// task.restoreDist()//恢复小区status=0
|
||||
|
||||
task.runMatch()
|
||||
// task.runMatch()
|
||||
}
|
||||
|
||||
def addOldMatch(){
|
||||
//老匹配
|
||||
def fileName = "D:\\数据\\国信达\\非正常数据.xlsx"//11城重新匹配.xlsx
|
||||
def fileName = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"//11城重新匹配.xlsx
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
println d['C']
|
||||
if(!d['AJ']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['AD'], d['H'], null, false, d['AE'], d['C'])
|
||||
if(!d['AJ'] && d['I']){
|
||||
def jsonObject = OpenApiClient.addressMatch(d['C'], d['D'], "district", false,null, d['D'])
|
||||
if (jsonObject != null && !jsonObject.containsKey("result")) {
|
||||
d['AJ'] = jsonObject.get("id")
|
||||
d['AK'] = jsonObject.get("city")
|
||||
|
||||
+347
@@ -0,0 +1,347 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 名录集
|
||||
*/
|
||||
class MinglujiData {
|
||||
|
||||
DBCollection mlj_COLL = MongoUtils.getCollectionByName("crawl", "kfq_mingluji_com", "192.168.0.206",27020)
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL)
|
||||
|
||||
DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
DBCollection community_100_CONN = MongoUtils.getCollectionByName2("fangjia", "community", MONGO100_URL)
|
||||
|
||||
|
||||
DBCollection company_100_CONN = MongoUtils.getCollectionByName2("fangjia", "company", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new MinglujiData()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def prov = "浙江省"
|
||||
def city = "嘉兴"
|
||||
// exportDatas(city)
|
||||
// matchDev(prov, city)
|
||||
matchCommunity(city)
|
||||
|
||||
// tempMatchDev(prov)
|
||||
|
||||
|
||||
// saveMljData()
|
||||
}
|
||||
def exportDatas(city){
|
||||
def datas = findMinglujiDatas(city)
|
||||
ExcelUtils.write("D:\\数据\\数据建设\\名录集\\${city}.xlsx", datas, "数据")
|
||||
}
|
||||
def matchDev(prov, city){
|
||||
def devs = loadAllDevs(prov)
|
||||
|
||||
def i = 1;
|
||||
while(i<20) {
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if (CommonDatautils.isWindow()) {
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
|
||||
}
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName, { d ->
|
||||
if (d['D']) {
|
||||
def street = d['D']
|
||||
def splitStreet = ""
|
||||
if (street) {
|
||||
if (street.contains(d['C'])) {
|
||||
def start = street.toString().lastIndexOf(d['C']) + d['C'].size()
|
||||
def end = street.toString().size()
|
||||
splitStreet = street.toString().substring(start, end)
|
||||
} else {
|
||||
splitStreet = street
|
||||
}
|
||||
}
|
||||
d['K'] = splitStreet
|
||||
}
|
||||
|
||||
if (d['I']) {
|
||||
def splitCommunity = ""
|
||||
if (d['C'] && d['I'].toString().contains(d['C'])) {
|
||||
splitCommunity = d['I'].toString().substring(d['I'].toString().lastIndexOf(d['C']) + d['C'].toString().size(), d['I'].toString().size())
|
||||
} else {
|
||||
splitCommunity = d['I']
|
||||
}
|
||||
d['L'] = splitCommunity
|
||||
}
|
||||
|
||||
|
||||
if (d['E'] && !d['N']) {
|
||||
def obj = findDevByName(d['E'])
|
||||
if (!obj) {
|
||||
obj = findDevByAlias(d['E'])
|
||||
if (!obj) {
|
||||
obj = findDebBySimlar(devs, d['E'], d['B'])
|
||||
if (!obj) {
|
||||
obj = findDevByPark(d['A'], d['B'], d['E'])
|
||||
if (!obj) {
|
||||
} else {
|
||||
d['N'] = "园区"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "相似"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "别名"
|
||||
}
|
||||
} else {
|
||||
d['N'] = "相同"
|
||||
}
|
||||
if (obj) {
|
||||
d['M'] = obj.name
|
||||
d['O'] = obj.city
|
||||
d['P'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
i++
|
||||
println fileName +" 完成"
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def tempMatchDev(prov){
|
||||
def devs = loadAllDevs(prov)
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州-开发区-等级.xlsx",{d->
|
||||
if(d['U'] && !d['W'] ){
|
||||
def obj = findDevByName(d['U'])
|
||||
if(!obj){
|
||||
obj = findDevByAlias(d['U'])
|
||||
if(!obj){
|
||||
obj = findDebBySimlar(devs, d['U'], d['B'])
|
||||
if(!obj){
|
||||
obj = findDevByPark(d['A'], d['B'], d['U'])
|
||||
if (!obj) {
|
||||
}else{
|
||||
d['X'] = "园区"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "相似"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "别名"
|
||||
}
|
||||
}else{
|
||||
d['X'] = "相同"
|
||||
}
|
||||
if(obj){
|
||||
d['W'] = obj.name
|
||||
d['Y'] = obj.city
|
||||
d['Z'] = obj.district
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def matchCommunity(city){
|
||||
def i = 1;
|
||||
while(i<20){
|
||||
def fileName = "/home/goojia/gj/mlj_${i}.xlsx"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "D:\\数据\\数据建设\\名录集\\${city}_${i}.xlsx"
|
||||
}
|
||||
|
||||
File file=new File(fileName);
|
||||
if(file.exists()){
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
if(d['A'] && d['B'] && d['L'] && !d['Q']){
|
||||
def list = findCommunityByName(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
list = findCommunityBySimlar(d['A'], d['B'], d['L'])
|
||||
if(!list){
|
||||
|
||||
}else{
|
||||
d['T'] = "包含"
|
||||
}
|
||||
}else{
|
||||
d['T'] = "相同"
|
||||
}
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
def obj = list[0]
|
||||
d['Q'] = obj.name
|
||||
d['R'] = obj.street_name
|
||||
d['U'] = obj.area_name
|
||||
d['V'] = obj.city_name
|
||||
}else if(list.size()>=2){
|
||||
for(def l : list){
|
||||
if(d['C']==l.area_name){
|
||||
d['Q'] = l.name
|
||||
d['R'] = l.street_name
|
||||
d['U'] = l.area_name
|
||||
d['V'] = l.city_name
|
||||
break;
|
||||
}
|
||||
}
|
||||
d['S'] = list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
println fileName +" 远行完成"
|
||||
}
|
||||
i++
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def saveMljData(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\名录集\\湖州入库数据.xlsx",{d->
|
||||
def obj = new BasicDBObject()
|
||||
obj.put("province", d['A'])
|
||||
obj.put("city", d['B'])
|
||||
obj.put("district", d['C'])
|
||||
obj.put("name", d['D'])
|
||||
obj.put("address", d['E'])
|
||||
obj.put("dev_name", d['F'])
|
||||
obj.put("committee", d['G'])
|
||||
obj.put("street", d['H'])
|
||||
company_100_CONN.save(obj)
|
||||
})
|
||||
}
|
||||
def loadAllDevs(prov){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province", prov)
|
||||
return development_zone_CONN.find(q).toList()
|
||||
}
|
||||
|
||||
def findDebBySimlar(def devs, def name, def city){
|
||||
name = name.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
def last = null
|
||||
for(def d : devs){
|
||||
def alias1 = null
|
||||
if(!d.alias){
|
||||
alias1 = []
|
||||
}else{
|
||||
alias1 = d.alias
|
||||
}
|
||||
if(!alias1.contains(d.name)){
|
||||
alias1 << d.name
|
||||
}
|
||||
if(alias1){
|
||||
try {
|
||||
for(def n : alias1){
|
||||
n = n.replace("经济开发区", "经济技术开发区").replace("工业园区", "工业园")
|
||||
if(d.city && d.city.contains(city) && (n.contains(name) || name.contains(n))){
|
||||
last = d
|
||||
break
|
||||
}
|
||||
}
|
||||
}catch(exception){
|
||||
println alias1
|
||||
}
|
||||
|
||||
}
|
||||
if(last){
|
||||
break
|
||||
}
|
||||
}
|
||||
return last
|
||||
}
|
||||
def findDevByName(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("name", name)
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByAlias(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('alias', new BasicDBObject('$in',[name]))
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevBySimlar(name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
return development_zone_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findDevByPark(prov, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put('province',new BasicDBObject('$regex', Pattern.compile('^.*'+prov+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('city',new BasicDBObject('$regex', Pattern.compile('^.*'+city+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put('park',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
q.put("type", "园区")
|
||||
return dev_mapping_CONN.findOne(q)
|
||||
}
|
||||
|
||||
def findCommunityByName(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", name)
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findCommunityBySimlar(province, city, name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("province_name", province+"省")
|
||||
q.put("city_name", city+"市")
|
||||
q.put("name", new BasicDBObject('$regex', Pattern.compile('^'+name+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def dbcursor = community_100_CONN.find(q)
|
||||
def list = []
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
def findMinglujiDatas(city){
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("c_address",1)
|
||||
keys.put("c_city",1)
|
||||
keys.put("c_committee",1)
|
||||
keys.put("c_community",1)
|
||||
keys.put("c_company",1)
|
||||
keys.put("c_district",1)
|
||||
keys.put("c_street",1)
|
||||
keys.put("kind",1)
|
||||
keys.put("name",1)
|
||||
keys.put("province",1)
|
||||
def datas = mlj_COLL.find(new BasicDBObject("c_city", city), keys)
|
||||
def rets = []
|
||||
rets << ["省","城市","区域","街道","开发区名","类型","公司名","社区","居委","地址","拆分街道","拆分居委","匹配开发区名"]
|
||||
for(def d : datas){
|
||||
def street = d.c_street
|
||||
def splitStreet = ""
|
||||
if(street){
|
||||
if(street.contains(d.c_district)){
|
||||
def start = street.toString().lastIndexOf(d.c_district)+d.c_district.size()
|
||||
def end = street.toString().size()
|
||||
splitStreet = street.toString().substring(start, end)
|
||||
}
|
||||
}
|
||||
def splitCommunity = ""
|
||||
if(d.c_committee){
|
||||
if(d.c_committee.contains(d.c_district)){
|
||||
splitCommunity = d.c_committee.substring(d.c_committee.lastIndexOf(d.c_district)+d.c_district.size(), d.c_committee.toString().size())
|
||||
}
|
||||
}
|
||||
rets << [d.province, d.c_city, d.c_district, d.c_street, d.name, d.kind, d.c_company, d.c_community, d.c_committee, d.c_address, splitStreet, splitCommunity]
|
||||
}
|
||||
return rets
|
||||
}
|
||||
}
|
||||
+88
-4
@@ -25,7 +25,7 @@ import java.util.concurrent.atomic.AtomicLong
|
||||
class PoiLocation {
|
||||
|
||||
def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi", MAP_POI_URL)
|
||||
DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "didata_map_poi_single", MAP_POI_URL)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
def regionMap = com.fangjia.script.commData.CommonDatautils.queryDistNameByCity()
|
||||
@@ -35,21 +35,52 @@ class PoiLocation {
|
||||
|
||||
static main(args) {
|
||||
def task = new PoiLocation();
|
||||
task.run();
|
||||
// task.run();
|
||||
task.regionAndStreetProcess()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
def citys = ["芜湖市","重庆市","杭州市","泉州市","东莞市","福州市","厦门市","佛山市","天津市","武汉市","宁波市","郑州市","长沙市","青岛市","上海市","西安市","温州市","无锡市","南京市","湖州市","合肥市","临沂市","石家庄市","昆明市","金华市","保定市","成都市","济南市","北京市","沈阳市","潍坊市","广州市","惠州市","嘉兴市","赣州市","苏州市","南通市","南宁市","台州市","南阳市","徐州市","长春市","深圳市","哈尔滨市","漳州市","中山市","常州市","烟台市","大连市","邯郸市","济宁市","盐城市","绍兴市","唐山市","洛阳市","阜阳市","菏泽市","贵阳市","沧州市","遵义市","莆田市","南昌市","商丘市","周口市","邢台市","江门市","衡阳市","邵阳市","信阳市","廊坊市","扬州市","泰州市","新乡市","太原市","德州市","淄博市","宁德市","绵阳市","汕头市","聊城市","淮安市","驻马店市","黄冈市","宿迁市","三明市","六安市","龙岩市","桂林市","岳阳市","南平市","揭阳市","上饶市","九江市","南充市","连云港市","襄阳市","泰安市","常德市","郴州市","安阳市","安庆市","湛江市","永州市","茂名市","曲靖市","宜宾市","肇庆市","宿州市","兰州市","运城市","镇江市","滁州市","威海市","梅州市","宜春市","清远市","亳州市","怀化市","荆州市","平顶山市","开封市","玉林市","咸阳市","宜昌市","株洲市","乌鲁木齐市","滨州市","渭南市","临汾市","榆林市","银川市","衡水市","毕节市","枣庄市","吉安市","张家口市","柳州市","许昌市","泸州市","呼和浩特市","达州市","德阳市","赤峰市","珠海市","孝感市","益阳市","娄底市","乐山市","恩施土家族苗族自治州","焦作市","湘潭市","海口市","红河哈尼族彝族自治州","黔南布依族苗族自治州","河源市","汉中市","日照市","大理白族自治州","濮阳市","黔东南苗族侗族自治州","秦皇岛市","晋中市","宝鸡市","吉林市","包头市","蚌埠市","东营市","宣城市","韶关市","衢州市","十堰市","承德市","丽水市","眉山市","鞍山市","昭通市","凉山彝族自治州","长治市","抚州市","鄂尔多斯市","吕梁市","百色市","齐齐哈尔市","贵港市","潮州市","黔西南布依族苗族自治州","广安市","淮南市","河池市","大同市","铜仁市","阳江市","咸宁市","营口市","绥化市","马鞍山市","安康市","荆门市","忻州市","广元市","内江市","遂宁市","西宁市","大庆市","汕尾市","文山壮族苗族自治州","锦州市","延安市","巴中市","呼伦贝尔市","楚雄彝族自治州","黄石市","玉溪市","六盘水市","通辽市","钦州市","云浮市","梧州市","漯河市","晋城市","庆阳市","安顺市","丹东市","自贡市","朝阳市","葫芦岛市","天水市","三门峡市","湘西土家族苗族自治州","北海市","陇南市","普洱市","延边朝鲜族自治州","保山市","定西市","随州市","喀什地区","黄山市","佳木斯市","巴彦淖尔市","牡丹江市","舟山市","资阳市","商洛市","松原市","锡林郭勒盟","伊犁哈萨克自治州","萍乡市","铁岭市","乌兰察布市","丽江市","景德镇市","张家界市","抚顺市","通化市","淮北市","来宾市","临沧市","阿克苏地区","贺州市","盘锦市","昌吉回族自治州","池州市","雅安市","三亚市","平凉市","四平市","酒泉市","西双版纳傣族自治州","鹤壁市","辽阳市","白城市","吴忠市","崇左市","阜新市","朔州市","白银市","拉萨市","巴音郭楞蒙古自治州","张掖市","铜陵市","武威市","临夏回族自治州","德宏傣族景颇族自治州","新余市","甘孜藏族自治州","兴安盟","本溪市","鄂州市","黑河市","阿坝藏族羌族自治州","阳泉市","防城港市","鸡西市","仙桃市","鹰潭市","固原市","白山市","中卫市","双鸭山市","攀枝花市","石嘴山市","和田地区","海东市","塔城地区","日喀则市","济源市","伊春市","天门市","辽源市","潜江市","那曲市","铜川市","海西蒙古族藏族自治州","鹤岗市","昌都市","阿拉善盟","乌海市","儋州市","哈密市","克拉玛依市","七台河市","金昌市","甘南藏族自治州","琼海市","迪庆藏族自治州","石河子市","阿勒泰地区","文昌市","玉树藏族自治州","嘉峪关市","万宁市","怒江傈僳族自治州","林芝市","海南藏族自治州","吐鲁番市","澄迈县","陵水黎族自治县","博尔塔拉蒙古自治州","山南市","阿里地区","大兴安岭地区","海北藏族自治州","东方市","乐东黎族自治县","克孜勒苏柯尔克孜自治州","果洛藏族自治州","临高县","阿拉尔市","黄南藏族自治州","神农架林区","屯昌县","定安县","昌江黎族自治县","五家渠市","保亭黎族苗族自治县","琼中黎族苗族自治县","五指山市","白沙黎族自治县","北屯市","图木舒克市","可克达拉市","双河市","铁门关市","毕节地区","昆玉市","铜仁地区","海东地区","三沙市","吐鲁番地区","哈密地区","日喀则地区","林芝地区","昌都地区","山南地区","那曲地区"]
|
||||
def citys = [""]
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
findPois(city)
|
||||
}
|
||||
}
|
||||
|
||||
def regionAndStreetProcess(){
|
||||
def citys = ['南平市','泉州市', '厦门市', '福州市', '龙岩市', '漳州市', '宁德市', '三明市', '莆田市']
|
||||
for(def city : citys){
|
||||
def datas = findAllPoisByCity(city)
|
||||
process(datas)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findAllPoisByCity(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
keys.put("province", 1)
|
||||
keys.put("city", 1)
|
||||
keys.put("district", 1)
|
||||
keys.put("township", 1)
|
||||
keys.put("location", 1)
|
||||
def dbcursor = MAPPOI_CONN.find(q).batchSize(5000).addOption(Bytes.QUERYOPTION_NOTIMEOUT)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def findPois(city){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("run_status", new BasicDBObject('$ne', 1))
|
||||
// q.put("run_status", new BasicDBObject('$ne', 1))
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
@@ -87,6 +118,32 @@ class PoiLocation {
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def process(datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
dataRun(d)
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
println d.city +" "+d.name
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
def checkData(def index, def datas){
|
||||
def totalCount = new AtomicLong()
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
@@ -190,6 +247,33 @@ class PoiLocation {
|
||||
return map;
|
||||
}
|
||||
|
||||
def dataRun(d){
|
||||
def regions = getFullRegion(d)
|
||||
if(d.location) {
|
||||
//判断区域否在围栏里
|
||||
def regionFlag = null
|
||||
if (d.district) {
|
||||
for (def rs : regions) {
|
||||
if (rs.shape && rs.shape.coordinates) {
|
||||
regionFlag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(rs.shape.coordinates))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (regionFlag) {//在围栏里
|
||||
//判断township是否在统计局中
|
||||
// if(d.township){
|
||||
// town = getSimilarTown(regions, d.city, d.township)
|
||||
// }
|
||||
|
||||
|
||||
}else{//不在转栏里,重新搜索
|
||||
|
||||
println(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def getFullRegion(d){
|
||||
def regions = regionMap.get(d.city)
|
||||
def names = []
|
||||
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThardParthAnalyBatch extends BaseBatch{
|
||||
|
||||
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
static DBCollection THIRD_PARTY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "third_party_stat", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThardParthAnalyBatch(), args)
|
||||
}
|
||||
|
||||
// static main(args) {
|
||||
// def task = new ThardParthAnalyBatch()
|
||||
// task.run()
|
||||
// }
|
||||
@Override
|
||||
void process(Object args) {
|
||||
run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def fjwTotalCount = CommonDatautils.findAllCount("district")
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
println "统计月份:"+sDate+" 查询更新时间:"+mdate
|
||||
|
||||
def crawlTotalCount = findAllEsDataCount(mdate)
|
||||
println "抓取第三方小区总量:"+crawlTotalCount
|
||||
def synchTotalCount = findAllMongoDatasCount(sDate)
|
||||
println "同步到第三方库小区总量: "+synchTotalCount
|
||||
|
||||
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
|
||||
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
|
||||
|
||||
def synchDistCount = findAllMongoDistrictCount(sDate)
|
||||
println "同步到第三方库有价格小区量: "+synchDistCount
|
||||
|
||||
def matchTotalCount = findMatchSuccessCount(sDate)
|
||||
println "第三方匹配成功总量:"+matchTotalCount
|
||||
|
||||
def matchDistTotalCount = findMatchSuccessDistCount(sDate)
|
||||
println "第三方匹配成功小区量:"+matchDistTotalCount
|
||||
|
||||
def precent = matchDistTotalCount*100.0/fjwTotalCount
|
||||
precent = DataHandleUtils.getDoubleRound(2, precent)
|
||||
println "匹配成功占比:"+precent+"%"
|
||||
|
||||
def noMatchTotalCount = findNoMatchCount(sDate)
|
||||
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
|
||||
|
||||
saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount);
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findAllEsDataCount(mdate){
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery("select count(1) as count from community where m_date>='${mdate}'", { d->
|
||||
count = d.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllEsDistDataCount(mdate){
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d->
|
||||
count = d.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllMongoDatasCount(s_date){
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findAllMongoDistrictCount(s_date){
|
||||
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("region",1)
|
||||
keys.put("name",1)
|
||||
def dbcursor = thirdPartyColl.find(q,keys)
|
||||
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findAllMongoDatasHavePricesCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findMatchSuccessCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findMatchSuccessDistCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findNoMatchCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){
|
||||
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fjw_dist_total", fjwTotalCount)
|
||||
udata.put("crawl_total", crawlTotalCount)
|
||||
udata.put("synch_total", synchTotalCount)
|
||||
udata.put("synch_have_price_total", synchHavePriceTotalCount)
|
||||
udata.put("synch_have_price_district_total", synchDistCount)
|
||||
udata.put("match_total", matchTotalCount)
|
||||
udata.put("match_dist_total", matchDistTotalCount)
|
||||
udata.put("match_dist_rate", precent)
|
||||
udata.put("no_match_total", noMatchTotalCount)
|
||||
udata.put("c_date", new Date())
|
||||
if(obj){
|
||||
udata.put("m_date", new Date())
|
||||
THIRD_PARTY_STAT.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
udata.put("s_date", sDate)
|
||||
THIRD_PARTY_STAT.save(udata)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+237
-2
@@ -2,8 +2,10 @@ package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -12,6 +14,8 @@ import com.mongodb.client.AggregateIterable
|
||||
import org.bson.Document
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
/**
|
||||
* 乡镇街道中间表
|
||||
* 包含市,区,乡镇街道别名,来源 统计局community表
|
||||
@@ -26,6 +30,9 @@ class TownMiddleProcess {
|
||||
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection ADMINISTRATIVE_MAPPING_43base_COLL = MongoUtils.getCollectionByName("fangjia_base", "administrative_mapping", "mongo.db.fangjia.com")
|
||||
|
||||
DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
|
||||
DBCollection CITYPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "city_polyline")
|
||||
DBCollection REGIONPOLYLINE_COLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
|
||||
@@ -67,7 +74,15 @@ class TownMiddleProcess {
|
||||
// checkTown()
|
||||
//区域民政部名称更新到乡镇街道中
|
||||
// updateTownshipByRegion()
|
||||
// addAliasFromExcel()
|
||||
|
||||
//增加简称及简拼
|
||||
// addSimpleName()
|
||||
|
||||
//根据民政部数据修改行政区划中间表
|
||||
// mzbDistrictUpate()
|
||||
|
||||
// addCity()
|
||||
}
|
||||
|
||||
|
||||
@@ -422,14 +437,16 @@ class TownMiddleProcess {
|
||||
for(def d : dbcursor){
|
||||
datas << d
|
||||
if (datas.size() >= 2000) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
println datas.size()
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
|
||||
if (datas.size() >= 0) {
|
||||
ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas);
|
||||
ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas)
|
||||
datas.clear();
|
||||
}
|
||||
}
|
||||
@@ -929,4 +946,222 @@ class TownMiddleProcess {
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(upquery, new BasicDBObject('$set',udata), false, true)
|
||||
}
|
||||
}
|
||||
|
||||
def addAliasFromExcel(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\乡镇街道合并或者变更20240627.xlsx",{ row->
|
||||
def q = new BasicDBObject("type", "乡镇街道")
|
||||
q.put("city", row['A'])
|
||||
q.put("street", row['D'])
|
||||
if(row['B']){
|
||||
q.put("district", row['B'])
|
||||
}
|
||||
def obj = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(obj){
|
||||
row['F'] = '是'
|
||||
def alias = obj.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(row['C'])){
|
||||
alias << row['C']
|
||||
}
|
||||
|
||||
if(alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", obj._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addSimpleName(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
for(def d : dbcursor){
|
||||
|
||||
def prov = ProvinceData.getSimpleProvince(d.province)
|
||||
if(prov){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("province_short", prov)
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", d._id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
// def city = d.city.replace("市","").replace("区","").replace("县","")
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("city_short", city)
|
||||
// if(d.district && d.district.size()>=3){
|
||||
// def region= d.district.replace("区","").replace("镇","").replace("乡","")
|
||||
// udata.put("district_short", region)
|
||||
// }
|
||||
// def updateQuery = new BasicDBObject("_id", d._id)
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
// def city = RegionData.formatCity(d.city)
|
||||
// if(city && d.mzb_district_name){
|
||||
// def region = RegionData.formatRegion(city, d.mzb_district_name)
|
||||
// if(region){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("city_short", city)
|
||||
// udata.put("district_short", region)
|
||||
//
|
||||
// def updateQuery = new BasicDBObject("_id", d._id)
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
// }
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def mzbDistrictUpate(){
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
|
||||
if(!row['I']){
|
||||
def updateType = row['E']
|
||||
def prov = row['B']
|
||||
def newStreet = row['G']
|
||||
|
||||
if(["新设街道","新设镇","新增街道","新建街道","新建乡镇","新设乡","新增乡","新增建制"].contains(updateType)){
|
||||
def dbo = findStreets(prov, newStreet, row['F'].toString().toDouble().toInteger().toString())
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['K'] = "存在"
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
}
|
||||
}else if(["撤乡设镇","撤乡建镇","撤乡设乡","撤乡、镇设镇","撤销设街道","撤销建街道","撤销街道建镇","撤镇设镇、街道","撤销镇新设镇","乡镇合并","整建制划归","撤镇建新镇","区划变更","撤镇、街道设镇","撤街道建镇","撤销乡、镇新设镇","撤镇、街道设街道","撤乡、街道设街道","撤乡设街道","撤乡建街道","撤镇设街道","撤镇建街道","名称变更","撤街道设镇","撤乡、镇设街道","撤街道、镇、乡","并入其他建制",'撤镇设镇',"合并","合并建镇","更名","撤销合并","合并建制","合并镇","合并街道","名称变更","合并调整","合并建街道","撤镇设街道、镇","撤乡、镇设乡","撤镇乡设街道","区划调整","合并乡","撤镇设市","区划代码调整","区划代码变更","撤销乡、镇设街道"].contains(updateType)){
|
||||
if(row['F']){
|
||||
def dbo = null
|
||||
def code = row['F'].toString()
|
||||
if(!code.contains(" ")){
|
||||
dbo = findStreets(prov, newStreet, code.toDouble().toInteger().toString())
|
||||
}
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
def alias = dbo.alias
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}
|
||||
if(row['D']){
|
||||
if(!alias.contains(row['D'])){
|
||||
row['K'] = "存在(加别名)"
|
||||
alias << row['D']
|
||||
addAlias(dbo._id, alias)
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
}else if(updateType == "撤街道设街道"){
|
||||
row['K'] = "人工处理"
|
||||
}else if(updateType == "隶属关系变更"){
|
||||
row['K'] = "人工处理"
|
||||
}else if(["撤销镇","撤乡","撤销街道","撤销","撤街道设社区工作站","撤销建制"].contains(updateType)){
|
||||
row['K'] = "人工处理"
|
||||
}else if(updateType =='撤销乡'){
|
||||
if(!row['F']){
|
||||
row['K'] = "人工处理"
|
||||
}
|
||||
}
|
||||
|
||||
if(prov && newStreet && (!row['K'] || (row['K'] && !row['K'].contains("存在")) )){
|
||||
def dbo = findStreets(prov, newStreet, null)
|
||||
if(dbo){
|
||||
row['I'] = dbo.city
|
||||
row['J'] = dbo.district
|
||||
row['K'] = "存在-编码不同"
|
||||
row['L'] = dbo.tjj_street_code
|
||||
row['M'] = dbo.pid
|
||||
def alias = dbo.alias
|
||||
if(!dbo.alias){
|
||||
alias = []
|
||||
}
|
||||
if(row['D']){
|
||||
if(!alias.contains(row['D'])){
|
||||
row['K'] = "存在(加别名)"
|
||||
alias << row['D']
|
||||
// addAlias(dbo._id, alias)
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}else{
|
||||
row['K'] = "存在"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def addCity(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\乡镇街道\\民政部\\2014-2022年.xlsx",{ row->
|
||||
def citycode = null
|
||||
if(row['F']){
|
||||
citycode = row['F'].toString().substring(0, 4)
|
||||
}else if(row['C']){
|
||||
citycode = row['C'].toString().substring(0, 4)
|
||||
}
|
||||
|
||||
if(citycode){
|
||||
def q = new BasicDBObject()
|
||||
q.put("mzb_city_code", citycode+"00")
|
||||
q.put("type", "城市")
|
||||
def dbo = ADMINISTRATIVE_MAPPING_COLL.findOne(q)
|
||||
if(dbo){
|
||||
row['O'] = dbo.city
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
def findStreets(prov, newStreet, streetCode){
|
||||
def q = new BasicDBObject()
|
||||
Pattern provPattern = Pattern.compile(prov+'.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("province", new BasicDBObject('$regex', provPattern))
|
||||
q.put("street", newStreet)
|
||||
def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(q)
|
||||
def list = []
|
||||
def dbo = null
|
||||
for(def d : dbcursor){
|
||||
list << d
|
||||
}
|
||||
if(list && list.size()==1){
|
||||
dbo = list[0]
|
||||
}else if(list && list.size()>1){
|
||||
if(streetCode){
|
||||
for(def d : list){
|
||||
if(d.tjj_street_code && d.tjj_street_code.contains(streetCode)){
|
||||
dbo = d
|
||||
break;
|
||||
}else if(streetCode.contains(d.pid)){
|
||||
dbo = d
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return dbo
|
||||
}
|
||||
|
||||
def addAlias(id, alias){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("alias", alias)
|
||||
udata.put("operator","高娟-批量")
|
||||
|
||||
def updateQuery = new BasicDBObject("_id", id)
|
||||
ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata))
|
||||
}
|
||||
}
|
||||
|
||||
+243
-11
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
@@ -9,12 +10,14 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.rabbitmq.client.impl.AMQImpl
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class TownshipStrandUpdate {
|
||||
|
||||
@@ -26,13 +29,26 @@ class TownshipStrandUpdate {
|
||||
def polylineMap = null
|
||||
def newAdvMappingMap = null
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
public static final DBCollection REGION_BLOCK_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com")
|
||||
|
||||
|
||||
|
||||
def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
|
||||
DBCollection dev_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL)
|
||||
|
||||
static main(args) {
|
||||
def task = new TownshipStrandUpdate()
|
||||
// task.process()
|
||||
task.updateTown()
|
||||
// task.updateTown()
|
||||
// task.stat()
|
||||
|
||||
// task.updateDiffRegionTown()
|
||||
// task.cleanTown()
|
||||
// task.checkTownFromMongo()
|
||||
// task.updateTownMapping()
|
||||
// task.updateDevMapping()
|
||||
println new Date().getTime()
|
||||
// task.updateSpecCityTown()
|
||||
}
|
||||
|
||||
|
||||
@@ -43,21 +59,21 @@ class TownshipStrandUpdate {
|
||||
polylineMap = CommonDatautils.findAllTownRanges()
|
||||
newAdvMappingMap = CommonDatautils.queryNewTownByCity()
|
||||
// def citys = ["鉗�","벳덱","柯소鏤","괏땍","해뽈돎","읜렌","꾕鹵","蠟소왯","뷜彊","饉憩","넓돠"]//CommonDatautils.findAllCity()
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
def citys = ["澄迈","临高"]//CommonDatautils.findAllCity(),"东方","澄迈","临高"
|
||||
citys = citys.reverse()
|
||||
for(def city : citys){
|
||||
//
|
||||
if(["中山","东莞"].contains(city)){
|
||||
continue
|
||||
}
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
// if(["中山","东莞"].contains(city)){
|
||||
// continue
|
||||
// }
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\写字楼\\写字楼乡镇街道格式化-${city}.xlsx"
|
||||
File file=new File(fileName);
|
||||
if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "shop", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
// if(!file.exists()){
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city","region","name","address","cat","estate_type2","status","township","lng","lat"])
|
||||
dists = checkData(dists)
|
||||
def datas = createDatas(dists)
|
||||
ExcelUtils.write(fileName, datas, "${city}")
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -162,6 +178,10 @@ class TownshipStrandUpdate {
|
||||
def running(d){
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
if(!regions && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(fullCity.city)){
|
||||
regions = []
|
||||
regions << fullCity.city
|
||||
}
|
||||
def gaodeMap = null
|
||||
def town = null
|
||||
def mapTown = null
|
||||
@@ -217,6 +237,10 @@ class TownshipStrandUpdate {
|
||||
}
|
||||
if( gaodeMap.district && gaodeMap.district!='[]'){
|
||||
mapRegion = gaodeMap.district
|
||||
}else{
|
||||
if(mapCity && ["中山市","东莞市","济源市","仙桃市","潜江市","天门市","神农架市","五指山市","琼海市","文昌市","万宁市","东方市","定安市","屯昌市","澄迈市","临高市","白沙市","昌江市","乐东市","陵水市","保亭市","琼中市","阿拉尔市","图木舒克市","五家渠市","北屯市","铁门关市","双河市","可克达拉市"].contains(mapCity)){
|
||||
mapRegion = mapCity
|
||||
}
|
||||
}
|
||||
if(gaodeMap.township && gaodeMap.township!='[]'){
|
||||
mapTownship = gaodeMap.township
|
||||
@@ -331,7 +355,7 @@ class TownshipStrandUpdate {
|
||||
def names = []
|
||||
if(regions){
|
||||
for(def r : regions){
|
||||
if(r.mzb_district_name.contains(d.region)){
|
||||
if(r.mzb_district_name.contains(d.region) || r.district_short.contains(d.region)){
|
||||
names << r.district
|
||||
}
|
||||
}
|
||||
@@ -381,7 +405,7 @@ class TownshipStrandUpdate {
|
||||
}
|
||||
}
|
||||
}
|
||||
if(district || count>10){
|
||||
if(township || district || count>10){
|
||||
stop = true
|
||||
}else{
|
||||
count ++
|
||||
@@ -397,4 +421,212 @@ class TownshipStrandUpdate {
|
||||
map.format_addr = format_addr
|
||||
return map;
|
||||
}
|
||||
|
||||
def updateDiffRegionTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\shop-不同区域.xlsx"
|
||||
ExcelUtils.multiThreadingModify(fileName,{d->
|
||||
def lists = CommonDatautils.findDistrictList(d['B'], null, d['D'], "shop")
|
||||
d['AA'] = lists.size()
|
||||
//只处理库中只有一个的
|
||||
if(lists && lists.size()==1 && d['Z']=="包含" && !d['AB']){
|
||||
def newRegion = RegionData.formatRegion(d['B'], d['M'])
|
||||
if(newRegion){
|
||||
if(d['X'] && d['X'].toString().contains(d['M']+" "+d['N'])){ //地图街道在中间表中
|
||||
def dbo = lists[0]
|
||||
def arrs = d['X'].split(" ")
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("region", newRegion)
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", arrs[1])
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "shop", "update","高娟")
|
||||
d['AB'] = "已处理"
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def cleanTown(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\district-不同区域.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['AC']){
|
||||
def dbo = CommonDatautils.findDistrict(d['B'], d['C'], d['D'], "district")
|
||||
if(dbo && dbo.township){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", "")
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(dbo.city, dbo.region, dbo.name, "office", "update-town","高娟")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def checkTownFromMongo(){
|
||||
def citys = CommonDatautils.findAllCity()
|
||||
fullCityMap = CommonDatautils.queryFullNameByCity()
|
||||
regionMap = CommonDatautils.queryDistNameByCity()
|
||||
commMap = CommonDatautils.queryCommunityByCity()
|
||||
citys = citys.reverse()
|
||||
def index = 0
|
||||
for(def city : citys) {
|
||||
index++
|
||||
if (["中山", "东莞"].contains(city)) {
|
||||
continue
|
||||
}
|
||||
|
||||
def dists = CommonDatautils.findAllDistrictsFromMg(city, "office", ["city", "region", "name", "address", "cat", "estate_type2", "status", "township", "lng", "lat"])
|
||||
for (def d : dists) {
|
||||
if (d.township) {
|
||||
def fullCity = fullCityMap.get(d.city)
|
||||
def regions = getFullRegion(fullCity, d)
|
||||
def town = getSimilarTown(regions, fullCity, d.township)
|
||||
if (!town) {
|
||||
town = findStreet(d.city, d.city, d.township)
|
||||
if (!town) {
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", "")
|
||||
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',d._id), new BasicDBObject('$set', udata))
|
||||
CommonDatautils.saveOptLog(d.city, d.region, d.name, "office", "update-town","高娟")
|
||||
|
||||
println d.city + " " + d.region + " " + d.township + " " + d.name
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
println index + " " + city + " 完成"
|
||||
}
|
||||
}
|
||||
|
||||
def findStreet(city, district, street){
|
||||
def q = new BasicDBObject()
|
||||
Pattern provPattern = Pattern.compile(city+'.*$', Pattern.CASE_INSENSITIVE);
|
||||
q.put("city", new BasicDBObject('$regex', provPattern))
|
||||
q.put("district", new BasicDBObject('$regex', Pattern.compile(district+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
q.put("street", new BasicDBObject('$regex', Pattern.compile(street+'.*$', Pattern.CASE_INSENSITIVE)))
|
||||
def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList()
|
||||
if(list){
|
||||
if(list.size()==1){
|
||||
return list[0]
|
||||
}else{
|
||||
println city+ " "+district+" "+street+" 有多个"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def updateTownMapping(){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\双方区域对比.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
if(d['F']){
|
||||
// def udata = new BasicDBObject()
|
||||
// udata.put("mzb_district_name", d['F'])
|
||||
//
|
||||
// def updateQuery = new BasicDBObject()
|
||||
// updateQuery.put("city", d['C'])
|
||||
// updateQuery.put("district", d['D'])
|
||||
// ADMINISTRATIVE_MAPPING_COLL.update(updateQuery, new BasicDBObject('$set',udata), false, true)
|
||||
|
||||
def city = RegionData.formatCity(d['C'])
|
||||
if(city){
|
||||
def region = RegionData.formatRegion(city, d['D'])
|
||||
if(region){
|
||||
def updateQuery = new BasicDBObject()
|
||||
updateQuery.put("category", "region")
|
||||
updateQuery.put("city", city)
|
||||
updateQuery.put("name", region)
|
||||
|
||||
def list = REGION_BLOCK_COLL.find(updateQuery).toList()
|
||||
if(list && list.size()==1){
|
||||
def dbo = list[0]
|
||||
def query = new BasicDBObject()
|
||||
query.put("_id", dbo._id)
|
||||
|
||||
// def pattern = dbo.pattern
|
||||
// if(pattern.contains(d['F'])){
|
||||
// pattern.remove(d['F'])
|
||||
// }
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("org_name", d['F'])
|
||||
// udata.put("pattern",pattern)
|
||||
REGION_BLOCK_COLL.update(query, new BasicDBObject('$set',udata), false, true)
|
||||
}else{
|
||||
println city +" "+region+" "+list.size()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateDevMapping(){
|
||||
def fileName = "D:\\数据\\数据建设\\开发区\\开发区映射表重复数据.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", d['A'])
|
||||
q.put("district", d['B'])
|
||||
q.put("name",d['C'])
|
||||
q.put("type", "乡镇街道")
|
||||
q.put("township", d['D'])
|
||||
def list = dev_MAPPING_COLL_100.find(q).toList()
|
||||
if(list && list.size()==2){
|
||||
def obj = list[0]
|
||||
def removeQuery = new BasicDBObject()
|
||||
removeQuery.put("_id", obj._id)
|
||||
dev_MAPPING_COLL_100.remove(removeQuery)
|
||||
}else{
|
||||
println q
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def updateSpecCityTown(){
|
||||
def citys = ["中山","东莞","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉"]
|
||||
|
||||
// def citys = ["中山"]
|
||||
for(def city : citys){
|
||||
def fileName = "D:\\数据\\数据建设\\乡镇街道\\商铺\\商铺乡镇街道格式化-${city}.xlsx"
|
||||
ExcelUtils.modify(fileName,{d->
|
||||
def list = CommonDatautils.findDistrictList(d['A'], null , d['C'], "office")
|
||||
if(list.size()>1){
|
||||
println d['A'] +" "+d['C'] + " 有多个"
|
||||
}else if(list.size()==1){
|
||||
def dbo = list[0]
|
||||
def town = null
|
||||
// if(d['X']){
|
||||
// town = d['X']
|
||||
// }else
|
||||
if(d['R']){
|
||||
town = d['R']
|
||||
}
|
||||
if(!town && d['M']){
|
||||
town = d['M']
|
||||
}
|
||||
def region = RegionData.formatRegion(d['A'], town)
|
||||
if(region){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", region)
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
udata.put("township", town)
|
||||
udata.put("region", region)
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',dbo._id), new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
println d['A']+" "+d['C']+" "+d['X'] +" 无法格式化"
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ class FjwBigdataCommbuildtypeBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
|
||||
+12
-10
@@ -33,7 +33,7 @@ class FjwBigdataCommunityBatch extends BaseBatch{
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
//235为房价网内部计算均价的库
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
@@ -53,31 +53,33 @@ class FjwBigdataCommunityBatch extends BaseBatch{
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def index = 0
|
||||
def today = new Date().getDate()
|
||||
|
||||
def fangjiaDists = new HashSet()
|
||||
println "starting 。。。。。。。。。"
|
||||
for(def city : citys){
|
||||
index ++
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"], "all")
|
||||
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
|
||||
def ccbMaps = findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用
|
||||
|
||||
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets)
|
||||
rets = addDeleteDist(dists, ccbMaps, rets, fangjiaDists)
|
||||
rows.addAll(rets)
|
||||
println "[${index}] "+city +" 小区量:"+dists.size()+" 总:"+rets.size()
|
||||
}
|
||||
|
||||
if(today > 10){
|
||||
// if(today > 10){
|
||||
delPgData()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
// }
|
||||
}
|
||||
|
||||
def addDeleteDist(dists, ccbMaps, rets){
|
||||
def fangjiaDists = []
|
||||
for(def d : dists){
|
||||
fangjiaDists << d._id.toString()
|
||||
}
|
||||
def addDeleteDist(dists, ccbMaps, rets, fangjiaDists){
|
||||
|
||||
println "房价网小区:"+fangjiaDists.size()+" 建行小区:"+rets.size()
|
||||
def ccbDists = ccbMaps.keySet()
|
||||
for(def ccbId : ccbDists){
|
||||
|
||||
+5
-6
@@ -40,7 +40,7 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
@@ -515,13 +515,12 @@ class FjwBigdataDealBatch extends BaseBatch{
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
|
||||
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
|
||||
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20200101", "yyyyMMdd"))
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
// def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20240501", "yyyyMMdd"))
|
||||
// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20240701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
|
||||
+9
-8
@@ -67,12 +67,12 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM");
|
||||
def curMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM");
|
||||
|
||||
// 最大重试次数
|
||||
private static final int MAX_TRY_COUNT = 5;
|
||||
@@ -93,10 +93,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
def endDate = curMonth+"19" //正式运行时curMonth要改
|
||||
println "房源时间:"+startDate+" "+endDate
|
||||
|
||||
def houseIdsfilter = findAllHouseIdsByCity()
|
||||
def houseIdsfilter = null;//findAllHouseIdsByCity()
|
||||
|
||||
for(def city : citys){
|
||||
index++
|
||||
houseIdsfilter = findAllHouseIdsByCity(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"], "all");
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, startDate.toInteger(), endDate.toInteger(), houseIdsfilter);
|
||||
@@ -491,9 +492,9 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
|
||||
def findHousesByMdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, rets1){
|
||||
def date = "20200101"//DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern("20200101","yyyyMMdd")//DateUtils.parseFromTightByPattern(date,"yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern("20240518", "yyyyMMdd")//DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")
|
||||
def date = DateUtils.addMonth(endDate.toString(), -3, "yyyyMMdd")
|
||||
def startMdate = DateUtils.parseFromTightByPattern(date,"yyyyMMdd")//DateUtils.parseFromTightByPattern("20240519","yyyyMMdd")
|
||||
def endMdate = DateUtils.parseFromTightByPattern(endDate.toString(), "yyyyMMdd")//DateUtils.parseFromTightByPattern("20240718", "yyyyMMdd")
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(date.toInteger()).to(endDate).includeLower(false).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
@@ -918,11 +919,11 @@ class FjwBigdataForsaleBatch extends BaseBatch{
|
||||
return houseIds;
|
||||
}
|
||||
|
||||
def findAllHouseIdsByCity(){
|
||||
def findAllHouseIdsByCity(cityid){
|
||||
// 创建布隆过滤器对象
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
// def houseIds = new HashSet()
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale",{it->
|
||||
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.bigdata_forsale where urbn_id='${cityid}'",{it->
|
||||
filter.put(it.gujia_sell_id)
|
||||
});
|
||||
|
||||
|
||||
+13
-4
@@ -34,7 +34,7 @@ class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
|
||||
def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15";
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
|
||||
@@ -48,19 +48,24 @@ class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
def rows = []
|
||||
|
||||
FjwPgsqlCommUtils.deleteGuidedPrice(curDate)
|
||||
def priceIdsfilter = FjwPgsqlCommUtils.findGuiDingIdsByCity()
|
||||
for (def city : citys) {
|
||||
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
|
||||
//添加自己库中的人工指导价
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, "2024-04-01", "2024-05-01")
|
||||
|
||||
def end = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), 1)
|
||||
def start = DateUtils.add(DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd"), -90)
|
||||
def prices = CcbCommonUtils.findPersonPrice(city, start, end)
|
||||
if(prices){
|
||||
def distMap = getDistrictMap(city)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices)
|
||||
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter)
|
||||
rows.addAll(rets)
|
||||
println city + " " + rets.size()
|
||||
}
|
||||
}
|
||||
println "【房价网内部】人工指导价总计:"+rows.size()
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
@@ -118,11 +123,15 @@ class FjwGuiDingPriceBatch extends BaseBatch{
|
||||
return obj
|
||||
}
|
||||
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices){
|
||||
def createCityDatas(cityCodeDatas, distCodeMap, distMap, prices, priceIdsfilter){
|
||||
def rets = []
|
||||
if(prices){
|
||||
for(def ps : prices){
|
||||
def dist = distMap.get(ps.region+"##"+ps.name)
|
||||
if(priceIdsfilter.mightContain(ps._id.toString())){
|
||||
println ps._id.toString() +ps.region+"##"+ps.name+" 已存在"
|
||||
continue
|
||||
}
|
||||
if(dist){
|
||||
def ret = []
|
||||
def areaData = distCodeMap.get(dist.region)
|
||||
|
||||
+14
-1
@@ -1,7 +1,11 @@
|
||||
package com.fangjia.script.gj.fjwPriceBaseData
|
||||
|
||||
import com.google.common.hash.BloomFilter
|
||||
import com.google.common.hash.Funnels
|
||||
import groovy.sql.Sql
|
||||
|
||||
import java.nio.charset.StandardCharsets
|
||||
|
||||
/**
|
||||
* 房价网内部使用
|
||||
*/
|
||||
@@ -9,7 +13,7 @@ class FjwPgsqlCommUtils {
|
||||
|
||||
public static def database = "fangjia"
|
||||
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/"+database+"?stringtype=unspecified",
|
||||
public static def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/"+database+"?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
public static def findAllDistrictIdsByOracle(def cityid){
|
||||
@@ -35,5 +39,14 @@ class FjwPgsqlCommUtils {
|
||||
sqlInstance.executeUpdate(sql2)
|
||||
}
|
||||
|
||||
public static def findGuiDingIdsByCity(){
|
||||
// ´´½¨²¼Â¡¹ýÂËÆ÷¶ÔÏó
|
||||
BloomFilter<CharSequence> filter = BloomFilter.create(Funnels.stringFunnel(StandardCharsets.UTF_8), 200000000, 0.000001);
|
||||
sqlInstance.eachRow("SELECT gujia_fsid FROM base.guided_price",{it->
|
||||
filter.put(it.gujia_fsid)
|
||||
});
|
||||
return filter
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
+29
-151
@@ -35,11 +35,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12",
|
||||
"房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"]
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5432/fangjia?stringtype=unspecified",
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
|
||||
"postgres", "Goojia123456", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
@@ -50,20 +47,23 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
public void process(Object args) {
|
||||
|
||||
def citys = CcbCommonUtils.getRunningCitys(args)
|
||||
def stop = false
|
||||
def month = -1
|
||||
while(!stop){
|
||||
if(sDate<202004){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
println weekend.toString() +" "+sDate
|
||||
runDatas(citys)
|
||||
}
|
||||
month = month -1
|
||||
sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
|
||||
if(sDate == 201912){
|
||||
stop = true;
|
||||
}
|
||||
}
|
||||
runDatas(citys)
|
||||
|
||||
//ͬ²½ÀúÊ·Êý¾Ý
|
||||
// def stop = false
|
||||
// def month = -1
|
||||
// while(!stop){
|
||||
// if(sDate>202404 && sDate<202407){
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightByPattern(sDate+"15", "yyyyMMdd"))
|
||||
// println weekend.toString() +" "+sDate
|
||||
// runDatas(citys)
|
||||
// }
|
||||
// month = month -1
|
||||
// sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), month, "yyyyMM"));
|
||||
// if(sDate == 201912){
|
||||
// stop = true;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
def runDatas(citys){
|
||||
@@ -265,6 +265,10 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
if(third.get("5i5j_esf_price")){
|
||||
price_map.wiwj_avg_price = third.get("5i5j_esf_price")
|
||||
}
|
||||
|
||||
if(third.get("fccs_esf_price")){
|
||||
price_map.fccs_avg_price = third.get("fccs_esf_price")
|
||||
}
|
||||
dataMap.updatetime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
allRows << dataMap
|
||||
}
|
||||
@@ -311,6 +315,8 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}else if(k=="fccs_avg_price"){
|
||||
web = "·¿²ú³¬ÊÐ"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
@@ -389,139 +395,6 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
return rets;
|
||||
}
|
||||
|
||||
def createData3(cityid){
|
||||
def sql = "select COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, ZHUGE_AVG_PRICE, ANJUKE_AVG_PRICE, FANG_AVG_PRICE, LIANJIA_AVG_PRICE, CITYHOUSE_AVG_PRICE, MAITIAN_AVG_PRICE, KE_AVG_PRICE, TAOFANG_AVG_PRICE, ZHONGYUAN_AVG_PRICE, TBS_AVG_PRICE, QFANG_AVG_PRICE, FANGTAN_AVG_PRICE, FANGDUODUO_AVG_PRICE, WIWJ_AVG_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from THIRD_PARTY_PRICE_GJ where CITYID='${cityid}' and YEAR_MONTH=${sDate}";
|
||||
def allRows = []
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
def dataMap = [:]
|
||||
|
||||
dataMap.COMMID = r.COMMID
|
||||
dataMap.COMMNAME = r.COMMNAME
|
||||
dataMap.CITYID = r.CITYID
|
||||
dataMap.CITYNAME = r.CITYNAME
|
||||
dataMap.DISTID = r.DISTID
|
||||
dataMap.DISTNAME = r.DISTNAME
|
||||
dataMap.YEAR_MONTH = r.YEAR_MONTH
|
||||
def price_map = [:]
|
||||
dataMap.price_map = price_map
|
||||
if(r.ZHUGE_AVG_PRICE){
|
||||
price_map.zhuge_avg_price = r.ZHUGE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ANJUKE_AVG_PRICE){
|
||||
price_map.anjuke_avg_price = r.ANJUKE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANG_AVG_PRICE){
|
||||
price_map.fang_avg_price = r.FANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.LIANJIA_AVG_PRICE){
|
||||
price_map.lianjia_avg_price = r.LIANJIA_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.CITYHOUSE_AVG_PRICE){
|
||||
price_map.cityhouse_avg_price = r.CITYHOUSE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.MAITIAN_AVG_PRICE){
|
||||
price_map.maitian_avg_price = r.MAITIAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.KE_AVG_PRICE){
|
||||
price_map.ke_avg_price = r.KE_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TAOFANG_AVG_PRICE){
|
||||
price_map.taofang_avg_price = r.TAOFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.ZHONGYUAN_AVG_PRICE){
|
||||
price_map.zhongyuan_avg_price = r.ZHONGYUAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.TBS_AVG_PRICE){
|
||||
price_map.tbs_avg_price = r.TBS_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.QFANG_AVG_PRICE){
|
||||
price_map.qfang_avg_price = r.QFANG_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGTAN_AVG_PRICE){
|
||||
price_map.fangtan_avg_price = r.FANGTAN_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.FANGDUODUO_AVG_PRICE){
|
||||
price_map.fangduoduo_avg_price = r.FANGDUODUO_AVG_PRICE
|
||||
}
|
||||
|
||||
if(r.WIWJ_AVG_PRICE){
|
||||
price_map.wiwj_avg_price = r.WIWJ_AVG_PRICE
|
||||
}
|
||||
dataMap.updatetime = r.UPDATETIME
|
||||
|
||||
allRows << dataMap
|
||||
});
|
||||
|
||||
|
||||
def results = []
|
||||
for(def m : allRows){
|
||||
def priceMap = m.price_map
|
||||
priceMap.each{k,v->
|
||||
def ret = []
|
||||
ret << new ObjectId().toString()
|
||||
ret << m.CITYID
|
||||
ret << m.DISTID
|
||||
ret << m.COMMID
|
||||
ret << m.COMMNAME
|
||||
|
||||
def web = ""
|
||||
if(k=="zhuge_avg_price"){
|
||||
web = "诸葛找房"
|
||||
}else if(k=="anjuke_avg_price"){
|
||||
web = "安居客"
|
||||
}else if(k=="fang_avg_price"){
|
||||
web = "房天下"
|
||||
}else if(k=="lianjia_avg_price"){
|
||||
web = "链家"
|
||||
}else if(k=="cityhouse_avg_price"){
|
||||
web = "禧泰数据"
|
||||
}else if(k=="maitian_avg_price"){
|
||||
web = "麦田"
|
||||
}else if(k=="ke_avg_price"){
|
||||
web = "贝壳"
|
||||
}else if(k=="taofang_avg_price"){
|
||||
web = "淘房365"
|
||||
}else if(k=="zhongyuan_avg_price"){
|
||||
web = "中原"
|
||||
}else if(k=="tbs_avg_price"){
|
||||
web = "兔博士"
|
||||
}else if(k=="qfang_avg_price"){
|
||||
web = "Q房"
|
||||
}else if(k=="fangtan_avg_price"){
|
||||
web = "房探"
|
||||
}else if(k=="fangduoduo_avg_price"){
|
||||
web = "房多多"
|
||||
}else if(k=="wiwj_avg_price"){
|
||||
web = "我爱我家"
|
||||
}
|
||||
|
||||
ret << sourceMap.get(web)
|
||||
ret << web
|
||||
ret << m.YEAR_MONTH
|
||||
ret << v
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << m.updatetime
|
||||
results << ret
|
||||
}
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
}
|
||||
|
||||
|
||||
def findThirdPartyPrice(def city, def statMap){
|
||||
def q = new BasicDBObject()
|
||||
@@ -592,6 +465,11 @@ class FjwThirdPartyPriceBatch extends BaseBatch {
|
||||
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
|
||||
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
|
||||
}
|
||||
|
||||
if(!dataMap.get("fccs_esf_price") && db.get("fccs_esf_price") && db.get("fccs_esf_price")>3000){
|
||||
dataMap.put("fccs_esf_price", db.get("fccs_esf_price"))
|
||||
}
|
||||
|
||||
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
|
||||
// map.put(key, db)
|
||||
// }
|
||||
|
||||
+24
-24
@@ -33,11 +33,11 @@ class AddPriceTest {
|
||||
task.run();
|
||||
}
|
||||
def run(){
|
||||
def filePath = "D:\\数据\\建行\\分期小区对比结果.xlsx"
|
||||
def filePath = "D:\\数据\\建行\\房产估值\\20240716-20240815 _业务反馈价格.xlsx"
|
||||
// addGXDPrice(filePath)
|
||||
// addCcbPrice(filePath)
|
||||
addCcbPrice(filePath)
|
||||
// addThirdPrice(filePath)
|
||||
addFjwPrice(filePath)
|
||||
// addFjwPrice(filePath)
|
||||
// exportPkPrice()
|
||||
// addRegion()
|
||||
// addTradePrice(filePath)
|
||||
@@ -167,14 +167,14 @@ class AddPriceTest {
|
||||
}
|
||||
|
||||
def addCcbPrice(filepath){
|
||||
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
ExcelUtils.multiThreadingModify(filepath,{ d->
|
||||
if(d['A']){
|
||||
// def cityId = cityCodeDataMap.get(d['N'])
|
||||
def map = findAvgPrice(d['G'], d['A'].toString().toDouble().toInteger())
|
||||
if(d['U']){
|
||||
def cityId = cityCodeDataMap.get(d['V'])
|
||||
def map = findAvgPrice(d['U'], cityId)
|
||||
if(map){
|
||||
// d['AK'] = map.avg_price
|
||||
d['P'] = map.calc_type
|
||||
d['F'] = map.avg_price
|
||||
// d['P'] = map.calc_type
|
||||
// d['AM'] = map.credit_value
|
||||
}
|
||||
}
|
||||
@@ -183,7 +183,7 @@ class AddPriceTest {
|
||||
|
||||
|
||||
def findAvgPrice(def commid, def cityid){
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' and gujiabds_clc_yrmo_val=202407 order by gujiabds_clc_yrmo_val desc limit 1"
|
||||
def map = [:]
|
||||
pgsqlInstance.eachRow(sql, { r ->
|
||||
map.avg_price = r.gujia_bds_clc_avprc
|
||||
@@ -272,23 +272,23 @@ class AddPriceTest {
|
||||
ExcelUtils.multiThreadingModify(path,{d->
|
||||
// if(!d['L']){
|
||||
// if(d['J'] instanceof String){
|
||||
def dist = CommonDatautils.findDistrictById(d['J'])
|
||||
def dist = CommonDatautils.findDistrictById(d['AJ'])
|
||||
if(dist){
|
||||
d['A'] = dist.city
|
||||
d['B'] = dist.region
|
||||
d['C'] = dist.name
|
||||
d['D'] = dist.status
|
||||
d['E'] = dist.name_cat
|
||||
d['F'] = dist.estate_type2
|
||||
d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
|
||||
d['H'] = dist.master_district_name
|
||||
// d['A'] = dist.city
|
||||
// d['B'] = dist.region
|
||||
d['E'] = dist.name
|
||||
// d['D'] = dist.status
|
||||
// d['E'] = dist.name_cat
|
||||
// d['F'] = dist.estate_type2
|
||||
// d['G'] = DateUtils.formatDateByPattern(dist.m_date,'yyyy-MM-dd')
|
||||
// d['H'] = dist.master_district_name
|
||||
// d['N'] = dist.region
|
||||
// def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
// if(dbo){
|
||||
// d['D'] = dbo.avg_price
|
||||
//// d['X'] = dbo.cal_type
|
||||
def dbo = CommonDatautils.queryFirstDistrictPrice(dist.city, dist.region, dist.name)
|
||||
if(dbo){
|
||||
d['G'] = dbo.avg_price
|
||||
// d['X'] = dbo.cal_type
|
||||
// d['E'] = dbo.avg_price_2b
|
||||
// }
|
||||
}
|
||||
}
|
||||
// }
|
||||
// }
|
||||
|
||||
+2
-2
@@ -27,8 +27,8 @@ class QiliangTest {
|
||||
// task.run()
|
||||
|
||||
def data = [
|
||||
'address': '宁德市福安市中兴西路16号',
|
||||
'type': '2' //1:对公 0:对私
|
||||
'address': '湖州市南太湖高新技术产业园区大港村陈家斗80号',
|
||||
'type': '1' //1:对公 0:对私
|
||||
]
|
||||
def s = task.getProxyResult(data);
|
||||
print(s);
|
||||
|
||||
+265
@@ -0,0 +1,265 @@
|
||||
package com.fangjia.script.gj.xiamen
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticUtils89 {
|
||||
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
// connectionProperties.setProperty("user","developer");
|
||||
// connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
|
||||
connectionProperties.setProperty("user","elastic");
|
||||
connectionProperties.setProperty("password","elasticsearch");//elasticsearch
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.89"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient89(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 180);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "elasticsearch"));//goojia123456
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
if(results.size()%100000==0){
|
||||
println "."+ new Date()
|
||||
}
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
}
|
||||
+198
-50
File diff suppressed because one or more lines are too long
+1
-1
@@ -129,7 +129,7 @@ class SeaweedDataSynch extends BaseBatch{
|
||||
def run(){
|
||||
|
||||
fullCityMap = CommonDatautils.queryFullNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCity()
|
||||
mcaDistMap = CommonDatautils.queryDistNameCityByMapping()
|
||||
|
||||
// priceMap = findAllDistrictSellPrice()
|
||||
def citys = CommonDatautils.findAllCity().reverse()
|
||||
|
||||
+3
-3
@@ -61,10 +61,10 @@ class DataStandardBatch extends BaseBatch{
|
||||
private static def districtMap = [:];
|
||||
private static def limit = 5000;
|
||||
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final SHORT_DATE = Integer.parseInt(com.fangjia.script.util.DateUtils.addMonth(com.fangjia.script.util.DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里"]
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋","幸福里","中国房价行情","常州住房"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
static {
|
||||
thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
@@ -922,7 +922,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
//查找出所有未进行匹配的数据
|
||||
def findAllNoMatchData(){
|
||||
DBObject qdata = new BasicDBObject();
|
||||
qdata.put("s_date", SHORT_DATE)
|
||||
qdata.put("s_date", new BasicDBObject('$gte', SHORT_DATE))
|
||||
// qdata.put("city","福州")
|
||||
// qdata.put("source","安居客")
|
||||
// qdata.put("name",Pattern.compile("龚家宅.*"))
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
|
||||
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
|
||||
// def esSources = ["搜房":"搜房"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里"]
|
||||
def esSources = ["我爱我家":"我爱我家","兔博士PC端":"兔博士","吉屋":"吉屋","贝壳找房":"贝壳找房","乐有家":"乐有家","365淘房":"365淘房","诸葛找房APP":"诸葛找房","链家":"链家","Q房":"Q房","麦田":"麦田","中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","C21":"C21","城市房产小程序":"城市房产","幸福里APP":"幸福里", "中国房价行情":"中国房价行情","常州住房":"常州住房"]
|
||||
|
||||
static main(args) {
|
||||
runBatch(new SynchDistrictPrice(), args);
|
||||
|
||||
+2
-1
@@ -32,7 +32,8 @@ class ThirdDataToGatherBatch extends BaseBatch{
|
||||
"房估估":"fanggugu_esf","诸葛找房":"zhugezhaofang_esf","安居客":"anjuke_esf","搜房":"soufang_esf","中原地产":"zhongyuan_esf","赶集":"ganji_esf",
|
||||
"链家":"lianjia_esf","兔博士":"tuboshi_esf","城市房产":"cityhouse_esf","贝壳找房":"beike_esf","Q房":"qfang_esf","Q房(新)":"qfang_new","中原地产(新)":"zhongyuan_new",
|
||||
"乐有家":"leyoujia_esf","乐有家(新)":"leyoujia_new","房探":"fangtan_esf","房多多":"fangdoudou_esf","小资家":"xiaozijia_esf","我爱我家":"5i5j_esf","麦田":"maitian_esf",
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf"
|
||||
"房途":"fangtoo_esf","C21":"c21_esf","乐居":"leju_esf","房产超市":"fccs_esf","58同城":"58tc_esf","365淘房":"365house_esf","吉屋":"jiwu_esf","幸福里":"xingfuli_esf",
|
||||
"中国房价行情":"creprice_esf","常州住房":"zwfc_esf"
|
||||
]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(2)
|
||||
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import javax.crypto.Cipher;
|
||||
import javax.crypto.KeyGenerator;
|
||||
import javax.crypto.SecretKey;
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
import java.util.Base64;
|
||||
|
||||
public class DESUtil {
|
||||
|
||||
private static final String ALGORITHM = "DES";
|
||||
|
||||
// 生成密钥
|
||||
public static SecretKey generateKey() throws Exception {
|
||||
KeyGenerator keyGenerator = KeyGenerator.getInstance(ALGORITHM);
|
||||
keyGenerator.init(56); // DES密钥长度为56位
|
||||
return keyGenerator.generateKey();
|
||||
}
|
||||
|
||||
// 将密钥转换为字符串
|
||||
public static String keyToString(SecretKey secretKey) {
|
||||
return Base64.getEncoder().encodeToString(secretKey.getEncoded());
|
||||
}
|
||||
|
||||
// 将字符串转换为密钥
|
||||
public static SecretKey stringToKey(String keyStr) {
|
||||
byte[] decodedKey = Base64.getDecoder().decode(keyStr);
|
||||
return new SecretKeySpec(decodedKey, 0, decodedKey.length, ALGORITHM);
|
||||
}
|
||||
|
||||
// DES加密
|
||||
public static String encrypt(String data, SecretKey secretKey) throws Exception {
|
||||
Cipher cipher = Cipher.getInstance(ALGORITHM);
|
||||
cipher.init(Cipher.ENCRYPT_MODE, secretKey);
|
||||
byte[] encryptedData = cipher.doFinal(data.getBytes());
|
||||
return Base64.getEncoder().encodeToString(encryptedData);
|
||||
}
|
||||
|
||||
// DES解密
|
||||
public static String decrypt(String encryptedData, SecretKey secretKey) throws Exception {
|
||||
Cipher cipher = Cipher.getInstance(ALGORITHM);
|
||||
cipher.init(Cipher.DECRYPT_MODE, secretKey);
|
||||
byte[] decodedData = Base64.getDecoder().decode(encryptedData);
|
||||
byte[] decryptedData = cipher.doFinal(decodedData);
|
||||
return new String(decryptedData);
|
||||
}
|
||||
|
||||
public static void main(String[] args) {
|
||||
try {
|
||||
// 生成密钥
|
||||
// SecretKey secretKey = generateKey();
|
||||
// String keyStr = keyToString(secretKey);
|
||||
// System.out.println("密钥: " + keyStr);
|
||||
//
|
||||
//// // 加密数据
|
||||
// String data = "高娟";
|
||||
// String encryptedData = encrypt(data, secretKey);
|
||||
// System.out.println("加密后的数据: " + encryptedData);
|
||||
|
||||
// 解密数据
|
||||
SecretKey newSecretKey = stringToKey("tnb12345");
|
||||
String decryptedData = decrypt("5E55C890CED08EFB7E3A05A0BAECB7B100D71965E4AD0FE2", newSecretKey);
|
||||
System.out.println("解密后的数据: " + decryptedData);
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -71,6 +71,10 @@ class MongoUtils {
|
||||
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.TEST_FANGJIA_MONGO_URI, db, name)
|
||||
}
|
||||
|
||||
if ((domainUrl == "192.168.0.206") && (port == 27020)) {
|
||||
return com.fangjia.core.util.MongoUtils.getDbCollection(Constants.SEAWATER_MONGO_URI, db, name)
|
||||
}
|
||||
|
||||
def key = db + name + domainUrl + port
|
||||
if (collectionMap.containsKey(key)) {
|
||||
return collectionMap[key]
|
||||
|
||||
+2
-1
@@ -12,7 +12,8 @@ import org.jsoup.Jsoup
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GaoDeApiUtils {
|
||||
def static GAO_DE_AK = "dfc2245bac8a7b2d5b62e060e59d4eca" //15021384995
|
||||
def static GAO_DE_AK = "c6fcd4bfbc40d73ee762b8115dca3390" //宝剑
|
||||
//15021384995:dfc2245bac8a7b2d5b62e060e59d4eca
|
||||
|
||||
/**
|
||||
* 冪帠僅蛌華硊<br>
|
||||
|
||||
+1
-1
@@ -1038,7 +1038,7 @@ public class EvaluationServiceImpl implements EvaluationService {
|
||||
|
||||
SqmetreRent sqmetreRent = sqmetreRentService.findNearestSqmetreRent(district.getCity(), district.getRegion(), district.getName(), DateUtil.formatToYYYYMMDD(weekend), "district");
|
||||
if(sqmetreRent!=null){
|
||||
long totalPrice = evaluationRateService.evaluationRent(sqmetreRent.getAvgPrice(), evaluation);
|
||||
long totalPrice = evaluationRateService.evaluationRent(sqmetreRent.getAvgPrice().intValue(), evaluation);
|
||||
|
||||
evaluation.setTotalPrice(((Number) (totalPrice)).intValue());
|
||||
|
||||
|
||||
+3
-3
@@ -194,7 +194,7 @@ public class HouseReportServiceImpl implements HouseReportService{
|
||||
SqmetreRent citySqmetreRent = sqmetreRentService.findNearestSqmetreRent(city, null, null, "city");
|
||||
RegionStat cityRegionStat = regionStatService.findByCity(city);
|
||||
if(null != citySqmetreRent && null != cityRegionStat){
|
||||
Double rate = DataUtil.getDoubleRound(2, citySqmetreRent.getAvgPrice()*12*100/cityRegionStat.getAvgPrice());
|
||||
Double rate = DataUtil.getDoubleRound(2, citySqmetreRent.getAvgPrice().intValue()*12*100/cityRegionStat.getAvgPrice());
|
||||
return rate;
|
||||
}
|
||||
return null;
|
||||
@@ -205,7 +205,7 @@ public class HouseReportServiceImpl implements HouseReportService{
|
||||
SqmetreRent regionSqmetreRent = sqmetreRentService.findNearestSqmetreRent(city, region, null, "region");
|
||||
RegionStat regionStat = regionStatService.findRegionStat(city, region);
|
||||
if(null != regionSqmetreRent && null != regionStat){
|
||||
Double rate = DataUtil.getDoubleRound(2, regionSqmetreRent.getAvgPrice()*12*100/regionStat.getAvgPrice());
|
||||
Double rate = DataUtil.getDoubleRound(2, regionSqmetreRent.getAvgPrice().intValue()*12*100/regionStat.getAvgPrice());
|
||||
return rate;
|
||||
}
|
||||
return null;
|
||||
@@ -224,7 +224,7 @@ public class HouseReportServiceImpl implements HouseReportService{
|
||||
distAvgPrice = statBo.getVillaDistrictStat().getAvgPrice();
|
||||
}
|
||||
if(distAvgPrice > 0){
|
||||
Double rate = DataUtil.getDoubleRound(2, distSqmetreRent.getAvgPrice()*12*100/distAvgPrice);
|
||||
Double rate = DataUtil.getDoubleRound(2, distSqmetreRent.getAvgPrice().intValue()*12*100/distAvgPrice);
|
||||
return rate;
|
||||
}
|
||||
}
|
||||
|
||||
+7
@@ -126,6 +126,13 @@ public class SqmetreRentServiceImpl implements SqmetreRentService{
|
||||
if(StringUtils.isNotBlank(cat)){
|
||||
params.add(new QueryParam("cat", cat));
|
||||
}
|
||||
if(null != startDate){
|
||||
params.add(new QueryParam("weekend",QueryParam.GREATER_THAN_EQUAL_OPERATOR, startDate));
|
||||
}
|
||||
|
||||
if(null != endDate){
|
||||
params.add(new QueryParam("weekend",QueryParam.LESS_THAN_EQUAL_OPERATOR, endDate));
|
||||
}
|
||||
|
||||
List<SortParam> sortParams = new ArrayList<SortParam>();
|
||||
sortParams.add(new SortParam("weekend", true));
|
||||
|
||||
+1
@@ -17,5 +17,6 @@ public interface VillageIdentifyService {
|
||||
int count(String villageType,String city1,String address);
|
||||
|
||||
List<VillageIdentify> findAll(String villageType,String city1,String address,Pager pager);
|
||||
List<VillageIdentify> findByCityAndAddress(String villageType,String city1,String address,Pager pager);
|
||||
|
||||
}
|
||||
|
||||
+17
@@ -62,6 +62,23 @@ public class VillageIdentifyServiceImpl implements VillageIdentifyService {
|
||||
return villageIdentifyDao.findAllByParams(queryParams, pager.getStartRow(), pager.getPageSize());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<VillageIdentify> findByCityAndAddress(String villageType, String city1, String address, Pager pager) {
|
||||
List<QueryParam> queryParams = new ArrayList<QueryParam>();
|
||||
if (StringUtils.isNotBlank(villageType)) {
|
||||
queryParams.add(new QueryParam("villageType", villageType));
|
||||
}else {
|
||||
queryParams.add(new QueryParam("villageType", "¶Ô¹«"));
|
||||
}
|
||||
if (StringUtils.isNotBlank(city1)) {
|
||||
queryParams.add(new QueryParam("cityParam", city1));
|
||||
}
|
||||
if (StringUtils.isNotBlank(address)) {
|
||||
queryParams.add(new QueryParam("address", address));
|
||||
}
|
||||
return villageIdentifyDao.findAllByParams(queryParams, pager.getStartRow(), pager.getPageSize());
|
||||
}
|
||||
|
||||
public void setVillageIdentifyDao(VillageIdentifyDao villageIdentifyDao) {
|
||||
this.villageIdentifyDao = villageIdentifyDao;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.service.open;
|
||||
|
||||
import java.util.Date;
|
||||
|
||||
public class StatAnalyBo {
|
||||
|
||||
private String key;
|
||||
@@ -13,6 +15,17 @@ public class StatAnalyBo {
|
||||
private Integer billingCount;
|
||||
private Integer purpose;
|
||||
private Double successRate;
|
||||
// 新增剩余使用次数和到期时间
|
||||
private Integer surplusCallCount;
|
||||
private Date deadline;
|
||||
|
||||
// 新增url和name
|
||||
private String url;
|
||||
|
||||
private String name;
|
||||
|
||||
// 添加日均使用量
|
||||
private Double dailyUse;
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
@@ -85,4 +98,44 @@ public class StatAnalyBo {
|
||||
public void setPurpose(Integer purpose) {
|
||||
this.purpose = purpose;
|
||||
}
|
||||
|
||||
public Integer getSurplusCallCount() {
|
||||
return surplusCallCount;
|
||||
}
|
||||
|
||||
public void setSurplusCallCount(Integer surplusCallCount) {
|
||||
this.surplusCallCount = surplusCallCount;
|
||||
}
|
||||
|
||||
public Date getDeadline() {
|
||||
return deadline;
|
||||
}
|
||||
|
||||
public void setDeadline(Date deadline) {
|
||||
this.deadline = deadline;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
|
||||
public void setUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public Double getDailyUse() {
|
||||
return dailyUse;
|
||||
}
|
||||
|
||||
public void setDailyUse(Double dailyUse) {
|
||||
this.dailyUse = dailyUse;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -481,5 +481,12 @@
|
||||
|
||||
@media(max-device-width:375px){
|
||||
.bg01{height: 1550px;}
|
||||
.label_title01 {font-size: 1.2rem;}
|
||||
.evaluationBox .evaluation_title {font-size: 1.2rem;}
|
||||
.sample_icon05 {background-size: 350px 200px;}
|
||||
}
|
||||
|
||||
@media(max-device-width:414px){
|
||||
.sample_icon05 {background-size: 400px 200px;}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user