This commit is contained in:
2024-03-01 10:26:00 +08:00
185 changed files with 20889 additions and 5967 deletions
+1 -1
View File
@@ -18,6 +18,6 @@
<finalName>goojia-auto</finalName>
</build>
<modules>
<module>../../projects/goojia-auto</module>
<!-- <module>../../projects/goojia-auto</module>-->
</modules>
</project>
File diff suppressed because it is too large Load Diff
@@ -459,6 +459,11 @@ public class BaseAvgImportServlet extends HttpServlet {
price.setCat("district");
price.setModifyDate(new Date());
guidingPriceService.save(price);
Map<String, Object> udata = new HashMap<String, Object>();
udata.put("modificationDate", new Date());
districtService.updateById(udata, dis.getId());
HttpRequest request = new HttpRequest();
HashMap<String, String> params = new HashMap<String, String>();
params.put("category", "sell");
@@ -10,7 +10,27 @@
<#include "/common_js.ftl" />
<script type="text/javascript">
</script>
<#-- 将表头固定-->
<style>
table {
width: 100%;
border-collapse: collapse;
table-layout: fixed;
}
th, td {
border: 1px solid #dddddd;
text-align: left;
padding: 8px;
}
th {
position: sticky;
top: 0;
background-color: #f2f2f2;
}
</style>
</head>
<body>
@@ -31,12 +51,13 @@
<div class="list">
<table width="50%">
<thead>
<tr>Ap
<tr>
API数量:&nbsp<b>${authorityBoList?size}</b>个
<th align="center" width="15%">
uri
</th>
<th align="center" width="15%">
描述
API名称
</th>
<th align="center" width="10%">
是否有权限<input type="checkbox" id="auth" >全选</input>
@@ -63,7 +84,7 @@
${bo.uri!}
</td>
<td align="center">
${bo.description!}
${bo.name!}
</td>
<td align="center">
<input type="checkbox" <#if bo.hasAuth>checked </#if> id="auth_${bo_index + 1}" ></input>
@@ -43,10 +43,20 @@
<input id="userId" name="userId" value="${userId!}" type="hidden"/>
<td>${apiUserBo.userName!}<#if (apiUserBo.displayName??)>(${apiUserBo.displayName!})</#if></td>
</tr>
<#-- <tr>-->
<#-- <td>密码</td>-->
<#-- <td><input id="resetPwd" type="button" value="重置密码"/></td>-->
<#-- </tr>-->
<#-- <tr>-->
<#-- <td><b>密码</b></td>-->
<#-- <td><input name="password" value="${apiUserBo.password!}" /></td>-->
<#-- </tr>-->
<tr>
<td>密码</td>
<td><input id="resetPwd" type="button" value="重置密码"/></td>
<td>
&nbsp;&nbsp;&nbsp;&nbsp;<a href="../apiUser/editPwd?userId=${apiUserBo.userId!}">修改密码</a>
</td>
</tr>
<tr>
<td>账户类型</td>
<td>
@@ -151,15 +161,15 @@
return re.test(strIP)
}
$("#resetPwd").click(function () {
if (confirm('确认重置密码为【123456】?')) {
$.get('resetPwd',
"userId=" + $("#userId").val(),
function (data) {
alert(status);
})
}
})
// $("#resetPwd").click(function () {
// if (confirm('确认重置密码为【123456】?')) {
// $.get('resetPwd',
// "userId=" + $("#userId").val(),
// function (data) {
// alert(status);
// })
// }
// })
</script>
</body>
</html>
@@ -0,0 +1,54 @@
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
<#compress>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN""http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<title>api用户密码修改</title>
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}"/>
<script type="text/javascript" src="../js/My97DatePicker/WdatePicker.js"></script>
<#include "/common_js.ftl" />
<script type="text/javascript">
</script>
</head>
<body>
<#include "/layout/header.ftl" />
<div class="main-content">
<div class="list">
<table width="50%">
<form id="submitForm" action="doEditPwd" method="post" >
<tr>
<td>用户名</td>
<input id="userId" name="userId" value="${userId!}" type="hidden"/>
<td>${apiUserBo.userName!}<#if (apiUserBo.displayName??)>(${apiUserBo.displayName!})</#if></td>
</tr>
<tr>
<td><b>新密码</b></td>
<td><input name="password" value="${apiUserBo.password!}" /></td>
</tr>
</form>
<tr>
<td></td>
<td><input onclick="submitForm();" type="reset" value="修改"/></td>
</tr>
</table>
</div>
</div>
<script type="text/javascript">
function submitForm() {
$("#submitForm").attr("action", "doEditPwd");
$("#submitForm").submit();
}
function isIP(strIP) {
if (!strIP)
return false;
var re = /^(\d+)(\.(\d+)){0,3}$/g //匹配IP地址的正则表达式
return re.test(strIP)
}
</script>
</body>
</html>
</#compress>
@@ -1,150 +1,179 @@
<#ftl attributes={"content_type":"text/html; charset=UTF-8"}>
<#compress>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<title>api用户列表</title>
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}" />
<#include "/common_js.ftl" />
</head>
<body>
<#include "/layout/header.ftl" />
<div class="main-content">
<font color="red">${msg!''}</font>
<div class="menuButton">
<a href="list" class="list">用户列表</a><a href="add" class="create">新建用户</a>
</div>
<hr/>
<div class="list">
<div style="padding-left:5px;">
共有用户<b>${apiUserBoList?size}</b>个
</div>
<table width="100%">
<thead>
<tr>
<th align="center" width="55px">
ID
</th>
<th align="center" width="100px">
昵称
</th>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<title>api用户列表</title>
<link rel="stylesheet" type="text/css" href="../style/main.css?v=${version}"/>
<#include "/common_js.ftl" />
<#-- 将表头固定-->
<style>
table {
width: 100%;
border-collapse: collapse;
table-layout: fixed;
}
th, td {
border: 1px solid #dddddd;
text-align: left;
padding: 8px;
}
th {
position: sticky;
top: 0;
background-color: #f2f2f2;
}
.resizable-td {
resize: horizontal; /* 允许水平调整大小 */
overflow: auto; /* 显示滚动条以便查看超出部分 */
min-width: 50px; /* 设置最小宽度,防止过于缩小 */
}
</style>
</head>
<body>
<#include "/layout/header.ftl" />
<div class="main-content">
<font color="red">${msg!''}</font>
<div class="menuButton">
<a href="list" class="list">用户列表</a><a href="add" class="create">新建用户</a>
</div>
<hr/>
<div class="list">
<div style="padding-left:5px;">
共有用户<b>${apiUserBoList?size}</b>个
</div>
<table width="100%">
<thead>
<tr>
<th align="center" width="55px">
ID
</th>
<th align="center" width="100px">
昵称
</th>
<th align="center" width="60px">
账户用途
</th>
<th align="center" width="100px">
username
</th>
<th align="center" width="60px">
password
</th>
<th align="center" width="100px">
username
</th>
<th align="center" width="60px">
password
</th>
<th align="center" width="100px">
销售负责人
</th>
<th align="center" width="100px">
公司
</th>
<th align="center" width="550px">
appKey
</th>
<th align="center" width="150px">
appKey
</th>
<th align="center" width="60px">
账户类型
</th>
<th align="center" width="60px">
账户用途
</th>
<th align="center" width="60px">
是否监控
</th>
<th align="center" width="60px">
是否监控
</th>
<th align="center" width="120px">
到期时间
</th>
<th align="center">
操作
</th>
</tr>
</thead>
<#if (apiUserBoList??&&apiUserBoList?size>0)>
<#list apiUserBoList as bo>
<tr <#if bo.status!=0>style="background-color:#C3C3C3"</#if>>
<td align="center">
${bo.userId!}
</td>
<td align="center">
<a href="../apiUser/edit?userId=${bo.userId!}">${bo.displayName!}</a>
</td>
<td>
<#if bo.purpose?? && bo.purpose == 1>
API
<#elseif bo.purpose?? && bo.purpose == 2>
PDS
<#elseif bo.purpose?? && bo.purpose == 4>
房生活
</#if>
</td>
<td align="center">
${bo.userName!}
</td>
<td align="center">
<#if bo.password?? && bo.password == t_util.md5Hex('123456')>
初始密码
<#else>
******
</#if>
</td>
<td align="center">
${bo.manager!}
</td>
<td align="center">
${bo.company!}
</td>
<td align="center">
${bo.appKey!}
</td>
<td align="center">
<#if bo.isTest==0>
正式账户
<#else>
测试账户
</#if>
</td>
<td>
<#if bo.purpose?? && bo.purpose == 1>
API
<#elseif bo.purpose?? && bo.purpose == 2>
PDS
<#elseif bo.purpose?? && bo.purpose == 4>
房生活
</#if>
</td>
<td>
<#if bo.isMonitor==1>
是
<#else>
否
</#if>
</td>
<td align="center">
<#if bo.deadTime??>
${bo.deadTime?string("yyyy年MM月dd日")}
</#if>
</td>
<td align="center">
<a href="../apiUser/auth?userId=${bo.userId!}">权限控制</a>
<a href="../apiUser/city?userId=${bo.userId!}">城市管理</a>
<a href="../apiUser/priceRating?userId=${bo.userId!}">价格评级</a>
<#if bo.status==0>
<a href="javascript:if(confirm('确认禁用?')){location.href='../apiUser/status?userId=${bo.userId!}&status=1'}">禁用</a>
<#else>
<a href="javascript:if(confirm('确认再次启用?')){location.href='../apiUser/status?userId=${bo.userId!}&status=0'}"><font style="color:#198B3D;">再启用</font></a>
</#if>
</td>
</tr>
</#list>
</#if>
</table>
</div>
</div>
</body>
</html>
<th align="center">
操作
</th>
</tr>
</thead>
<#if (apiUserBoList??&&apiUserBoList?size>0)>
<#list apiUserBoList as bo>
<tr <#if bo.status!=0>style="background-color:#C3C3C3"</#if>>
<td align="center">
${bo.userId!}
</td>
<td align="center">
<a href="../apiUser/edit?userId=${bo.userId!}">${bo.displayName!}</a>
</td>
<td>
<#if bo.purpose?? && bo.purpose == 1>
API
<#elseif bo.purpose?? && bo.purpose == 2>
PDS
<#elseif bo.purpose?? && bo.purpose == 4>
房生活
</#if>
</td>
<td align="center">
${bo.userName!}
</td>
<td align="center">
<#if bo.password?? && bo.password == t_util.md5Hex('123456')>
初始密码
<#else>
******
</#if>
</td>
<td align="center">
${bo.manager!}
</td>
<td align="center">
${bo.company!}
</td>
<td align="center" class="resizable-td">
${bo.appKey!}
</td>
<td align="center">
<#if bo.isTest==0>
正式账户
<#else>
测试账户
</#if>
</td>
<td>
<#if bo.purpose?? && bo.purpose == 1>
API
<#elseif bo.purpose?? && bo.purpose == 2>
PDS
<#elseif bo.purpose?? && bo.purpose == 4>
房生活
</#if>
</td>
<td>
<#if bo.isMonitor==1>
是
<#else>
否
</#if>
</td>
<td align="center">
<#if bo.deadTime??>
${bo.deadTime?string("yyyy年MM月dd日")}
</#if>
</td>
<td align="center">
<a href="../apiUser/auth?userId=${bo.userId!}">API权限控制</a>
<a href="../apiUser/auth?userId=${bo.userId!}&msg=0">新增API</a>
<a href="../apiUser/city?userId=${bo.userId!}">城市管理</a>
<a href="../apiUser/priceRating?userId=${bo.userId!}">价格评级</a>
<#if bo.status==0>
<a href="javascript:if(confirm('确认禁用?')){location.href='../apiUser/status?userId=${bo.userId!}&status=1'}">禁用</a>
<#else>
<a href="javascript:if(confirm('确认再次启用?')){location.href='../apiUser/status?userId=${bo.userId!}&status=0'}"><font
style="color:#198B3D;">再启用</font></a>
</#if>
</td>
</tr>
</#list>
</#if>
</table>
</div>
</div>
</body>
</html>
</#compress>
@@ -835,7 +835,20 @@ function add(){
alert('小区名不能为空!');
return false;
}
var estateType = $("#estateType2").val()
if(name.indexOf("商住")>-1){
if(estateType!="商住"){
alert('物业类型有误!');
return false;
}
}else{
if(name.indexOf("别墅")>-1){
if(estateType!="别墅"){
alert('物业类型有误!');
return false;
}
}
}
var alias=$("#alias").val();
if(alias){
var aliasArr=alias.toString().split("\n");
@@ -1148,6 +1148,20 @@ $(document).ready(function(){
alert('小区名不能为空!');
return false;
}
if(name.indexOf("商住")>-1){
if(estateTypeStr!="商住"){
alert('物业类型有误!');
return false;
}
}else{
if(name.indexOf("别墅")>-1){
if(estateTypeStr!="别墅"){
alert('物业类型有误!');
return false;
}
}
}
var alias=$("#alias").val();
@@ -8,6 +8,25 @@
.up {color: green;}
#spinner{width:500px;height:100px;margin:30px 0 0 100px;text-align:center;padding-top:50px;border:2px solid #CCC;}
#checkMsg{width:500px;margin:30px 0 0 100px;padding:10px 0 10px 50px;text-align:left;border:2px solid #CCC;}
<#-- 将表头固定-->
table {
width: 100%;
border-collapse: collapse;
table-layout: fixed;
}
th, td {
border: 1px solid #dddddd;
text-align: left;
padding: 8px;
}
th {
position: sticky;
top: 0;
background-color: #f2f2f2;
}
</style>
</head>
<body>
@@ -25,6 +25,7 @@ public class RegionData implements PriorityOrdered {
private final static Logger LOGGER = LoggerFactory.getLogger(RegionData.class);
private final static long REGION_RELOAD_INTERVAL = 1000;
private static DBCollection REGION_CONN = null;
private static DBCollection TOWN_CONN = null;
private static Set<String> ALL_CITY = new HashSet<String>();
private static Map<String,List<String>> cityAndAliasMap = new HashMap<String,List<String>>();
@@ -32,12 +33,15 @@ public class RegionData implements PriorityOrdered {
* 城市对应级别Map
*/
private static Map<String, Integer> cityGradeMap = new HashMap<String, Integer>();
protected final static Map<String, Map<String, Map<String, Map<String, List<String>>>>> townDataMap = new HashMap<String, Map<String, Map<String, Map<String, List<String>>>>>();
private static void init() {
try {
REGION_CONN = MongoUtils.getCollectionByName("fangjia", "region_block", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
TOWN_CONN = MongoUtils.getCollectionByName("fangjia", "township", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCollection SERVER_AREA = MongoUtils.getCollectionByName("fangjia", "server_area", MongoUtils.MongoDB_Domain_mongo, MongoUtils.MongoDB_Port_28017);
DBCursor cursor = SERVER_AREA.find(new BasicDBObject(), new BasicDBObject("display", 1).append("city_grade", 1).append("alias", 1));
BasicDBObject dbo;
while(cursor.hasNext()){
@@ -58,6 +62,7 @@ public class RegionData implements PriorityOrdered {
start(REGION_RELOAD_INTERVAL);
reload();
// reloadTown();
}
private static void start(long time) {
@@ -114,6 +119,66 @@ public class RegionData implements PriorityOrdered {
LOGGER.info("RegionData reload() size [" + regionDataMap.size() + "]");
}
public static void reloadTown(){
DBObject query = new BasicDBObject();
query.put("year", 2022);
DBObject fields = new BasicDBObject();
fields.put("fangjia_city", 1);
fields.put("fangjia_area", 1);
fields.put("street_name", 1);
fields.put("alias", 1);
DBCursor cursor = TOWN_CONN.find(query, fields);
while (cursor.hasNext()) {
DBObject dbo = cursor.next();
String city = (String) dbo.get("fangjia_city");
String region = (String) dbo.get("fangjia_area");
String street = (String) dbo.get("street_name");
List<String> ps = (List<String>) dbo.get("alias");
if(!ALL_CITY.contains(city)){
continue;
}
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
if(cityRegionMap == null){
cityRegionMap = new HashMap<String, Map<String, Map<String, List<String>>>>();
townDataMap.put(city, cityRegionMap);
}
Map<String, Map<String, List<String>>> regoinMap = cityRegionMap.get(region);
if(regoinMap == null){
regoinMap = new HashMap<String, Map<String, List<String>>>();
cityRegionMap.put(region, regoinMap);
}
Map<String, List<String>> townMap = regoinMap.get("street");
if(townMap == null){
townMap = new HashMap<String, List<String>>();
regoinMap.put(street, townMap);
}
if (CollectionUtils.isNotEmpty(ps) && ps.size() > 1) {
Collections.sort(ps, new Comparator<String>() {
public int compare(String arg0, String arg1) {
if (StringUtils.isBlank(arg0)) {
return 1;
}
if (StringUtils.isBlank(arg1)) {
return -1;
}
return arg1.length() - arg0.length();
}
});
}
if(null != ps && !ps.contains(street)){
ps.add(street);
}
townMap.put(street, ps);
}
LOGGER.info("TownData reload() size [" + townDataMap.size() + "]");
}
static class ScheduleRunTask extends TimerTask {
private int nextScheduleHour;
@@ -174,6 +239,28 @@ public class RegionData implements PriorityOrdered {
return ret;
}
public static String formatStreet(String city, String region, String street) {
if (StringUtils.isBlank(region) || StringUtils.isBlank(city) || StringUtils.isBlank(street)) {
return null;
}
String ret = null;
Map<String, Map<String, Map<String, List<String>>>> cityRegionMap = townDataMap.get(city);
if(MapUtils.isNotEmpty(cityRegionMap)){
Map<String, Map<String, List<String>>> streetMap = cityRegionMap.get(region);
if (MapUtils.isNotEmpty(streetMap)) {
for (Map<String, List<String>> crm : streetMap.values()) {
if (null != ret) {
break;
}
ret = format(crm, street);
}
}
}
return ret;
}
public static String parseRegionFormAddress(String city, String address){
if(StringUtils.isBlank(address)){
return null;
@@ -69,6 +69,7 @@ public class DataUtil {
private static final String hanziReg = "[\\u4e00-\\u9fa5]";
private static final String englishReg = "[A-Za-z]";
public static String[] TAGS =new String[]{"独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"};
public static String replaceAll(String original, String with, Pattern... pattern) {
if (StringUtils.isNotBlank(original)) {
@@ -1463,6 +1464,13 @@ public class DataUtil {
return false;
}
public static boolean isNumber(String s) {
if (s != null && !"".equals(s.trim()))
return s.matches("^[0-9]*$");
else
return false;
}
public static double roundDouble(double d, int scale) {
try {
double ret = new BigDecimal(String.valueOf(d)).setScale(scale, BigDecimal.ROUND_HALF_UP).doubleValue();
@@ -1834,6 +1842,44 @@ public class DataUtil {
return false;
}
/**
* 是否是别墅
* @return
*/
public static boolean isVilla(Double area, String name, Integer height, Integer room) {
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
boolean ret = false;
if (height != null && room != null) {
if (height <= 3) {
if (area > 200 && room >= 5) {
ret = true;
}
}
}else if(height != null){
if (height <= 3) {
if (area > 200){
ret = true;
}
}
} else if (height == null && room != null) {
if (area > 360 && room >= 6) {
ret = true;
}
} else if (height == null && room == null) {
if (area > 500) {
ret = true;
}
}
return ret;
}
@@ -165,6 +165,7 @@ public class SaleLeadsController {
}
saleLeads = converValue(saleLeads);
String body =
"业务需求:" + saleLeads.getBusiness() + " " + saleLeads.getOtherBusiness() + "\n\n"
@@ -363,11 +364,14 @@ public class SaleLeadsController {
if (saleLeads.getId()!=null) {
saleLeads.setId(null);
}
String ip = AnalyticsUtil.getIpAddress(req);
saleLeads = converValue(saleLeads);
String body =
"业务需求:" + saleLeads.getBusiness() + "\n\n"
+ "姓名:" + saleLeads.getUname() + "\n\n"
+ "手机:" + saleLeads.getMobile() + "\n\n"
+ "IP:"+ip + "\n\n"
+ "咨询内容:" + saleLeads.getRemarks()+ "\n\n" ;
body = body.replaceAll("null", "").replaceAll("其他,请填写", "");
if (StringUtils.isNotBlank(refUrl) && refUrl.contains("bd_vid")) {
@@ -99,6 +99,27 @@
@media(max-device-width:320px){
.searchBoxIndex{top:13.5rem;}
}
.advert {
position: fixed;
top: 30%;
left: 40%;
right: 0;
height: 320px;
width: 580px;
z-index: 10000;
}
.left-ad {
position: absolute;
bottom: 0;
left: 0;
width: 580px;
height: 300px;
background: url("${staticUrl}/images/fangjia/v5/adv.png");
background-size: 100% 100%;
}
.advert a{ float: right;margin-right: 10px;cursor: pointer;font-weight: bold;}
</style>
</head>
@@ -108,6 +129,10 @@
<#include "/header/v5/_header.ftl"/>
<#--<div class="advert mobile01">-->
<#-- <a id="close">关闭</a>-->
<#-- <div class="left-ad"></div>-->
<#--</div>-->
<div class="bannerIndex">
<div class="imgBoxIndex">
@@ -556,7 +581,18 @@ $.config = {router: false}
<script type="text/javascript" src="//developer.baidu.com/map/jsdemo/demo/convertor.js"></script>
<script src="${staticUrl}/js/fangjia/v5/city.js?v=${version}"></script>
<script>
// function closeWin(){
// $(".advert").addClass("dn")
// }
//
// $("#close").click(function(){
// closeWin();
// })
// window.setInterval(closeWin, 10000)
//
// $(".left-ad").click(function(){
// window.open('https://jinshuju.net/f/hBkMVo','_blank');
// })
/* banner轮播 */
$(".imgBoxIndex > div").hide();
$(".imgBoxIndex > div").eq(0).show();
@@ -0,0 +1,8 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.mongo.model.AdministrativeDivision;
import com.fangjia.dal.MongoDao;
public interface AdministrativeDivisionDao extends MongoDao<AdministrativeDivision> {
AdministrativeDivision findcityByCode(String code);
}
@@ -0,0 +1,23 @@
package com.fangjia.dal.mongo.dao;
import com.fangjia.dal.mongo.model.AdministrativeDivision;
import com.fangjia.dal.MongoDaoImpl;
import com.fangjia.dal.QueryParam;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
public class AdministrativeDivisionDaoImpl extends MongoDaoImpl<AdministrativeDivision> implements AdministrativeDivisionDao {
@Override
public AdministrativeDivision findcityByCode(String code) {
if (StringUtils.isBlank(code)) {
return new AdministrativeDivision();
}
List<QueryParam> params = new ArrayList<QueryParam>();
params.add(new QueryParam("mcaCityCode", QueryParam.EQUAL_OPERATOR, code));
params.add(new QueryParam("level", QueryParam.EQUAL_OPERATOR, "³ÇÊÐ"));
return (AdministrativeDivision) findFirstByParams(params);
}
}
@@ -0,0 +1,148 @@
package com.fangjia.dal.mongo.model;
import com.fangjia.dal.DatastoreCategory;
import com.fangjia.dal.MongoEntity;
import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;
@Entity
@Table(name = "fangjia.administrative_division")
public class AdministrativeDivision implements MongoEntity {
private static final long serialVersionUID = -8379446820760559477L;
private String id;
private String province;
private String city;
private String district;
private String mcaProvinceCode;
private String mcaDistrictCode;
private String mcaCityCode;
private String level;
private String cityShort;
private String distShort;
private String provShort;
private String code;
private String administer;
@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@Column(name = "_id")
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getProvince() {
return province;
}
public void setProvince(String province) {
this.province = province;
}
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
@Column(name = "mca_province_code")
public String getMcaProvinceCode() {
return mcaProvinceCode;
}
public void setMcaProvinceCode(String mcaProvinceCode) {
this.mcaProvinceCode = mcaProvinceCode;
}
@Column(name = "mca_district_code")
public String getMcaDistrictCode() {
return mcaDistrictCode;
}
public void setMcaDistrictCode(String mcaDistrictCode) {
this.mcaDistrictCode = mcaDistrictCode;
}
@Column(name = "mca_city_code")
public String getMcaCityCode() {
return mcaCityCode;
}
public void setMcaCityCode(String mcaCityCode) {
this.mcaCityCode = mcaCityCode;
}
@Column(name = "level")
public String getLevel() {
return level;
}
public void setLevel(String level) {
this.level = level;
}
@Column(name = "city_short")
public String getCityShort() {
return cityShort;
}
public void setCityShort(String cityShort) {
this.cityShort = cityShort;
}
@Column(name = "dist_short")
public String getDistShort() {
return distShort;
}
public void setDistShort(String distShort) {
this.distShort = distShort;
}
@Column(name = "prov_short")
public String getProvShort() {
return provShort;
}
public void setProvShort(String provShort) {
this.provShort = provShort;
}
public String getCode() {
return code;
}
public void setCode(String code) {
this.code = code;
}
public String getAdminister() {
return administer;
}
public void setAdminister(String administer) {
this.administer = administer;
}
}
@@ -184,7 +184,7 @@ public class DistrictFacilitys implements MongoEntity {
this.bus = bus;
}
@Column(name = "metro")
@Column(name = "subway")
public List<String> getMetro() {
return metro;
}
@@ -1,6 +1,8 @@
package com.fangjia.open.controllers;
import com.fangjia.core.util.HttpRequest;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.open.dto.addressGovernance.*;
import com.fangjia.open.interceptors.Authority;
import com.fangjia.open.interceptors.Role;
@@ -9,6 +11,9 @@ import com.fangjia.open.model.AddressModel;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.service.bo.DistrictStatBo;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.district.DistrictStatService;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
@@ -24,6 +29,9 @@ import java.util.Map;
@Authority(role = Role.USER)
public class AddrSvcController {
private DistrictStatService districtStatService;
private DistrictService districtService;
/**
* 地址结构化
*
@@ -179,7 +187,7 @@ public class AddrSvcController {
/**
* 地址匹配
*/
public Object matchAction(AddesssMatchModel model) {
public Object matchAction(AddesssMatchModel model) throws Exception {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
@@ -195,20 +203,13 @@ public class AddrSvcController {
params.put("category", model.getCategory());
params.put("district", model.getSuggestDistrict());
try {
// json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/match", params, "UTF-8");
//新匹配接口
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaNlpService/match", params, "UTF-8");
if (json != null) {
int code = json.getInt("status");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
basicModel.setResult(CommonDtoUtils.transformV(MatchedBuildingDto.class, json.get("result")));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
BasicModel basicModel = new BasicModel(200, "ok");
if (json != null && json.size() > 0) {
basicModel.setResult(CommonDtoUtils.transformV(MatchedBuildingDto.class, json));
}
mav.setModel(basicModel);
} catch (Exception e) {
e.printStackTrace();
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
@@ -263,4 +264,153 @@ public class AddrSvcController {
}
return mav;
}
public Object avgPriceAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", model.getCity());
params.put("address", model.getAddress());
params.put("category", model.getCategory());
params.put("useMap", String.valueOf(model.isUseMap()));
// params.put("district", model.getSuggestDistrict());//接口没有此参数
try {
//新匹配接口
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaNlpService/match", params, "UTF-8");
BasicModel basicModel = new BasicModel(200, "ok");
if (json != null && json.size() > 0) {
MatchedBuildingDto dto = CommonDtoUtils.transformV(MatchedBuildingDto.class, json);
District district = districtService.findByNameAndAlias(dto.getCity(), dto.getDistrict(), dto.getName(), "district");
if (null != district) {
DistrictStatBo statBo = districtStatService.findCurrentDistrictStat(new DistrictKey(dto.getCity(), dto.getDistrict(), dto.getName()));
if (null != statBo) {
if (statBo.getDistrictStat() != null) {
dto.setAvgPrice(statBo.getDistrictStat().getAvgPrice());
} else if (statBo.getVillaDistrictStat() != null) {
dto.setAvgPrice(statBo.getVillaDistrictStat().getAvgPrice());
}
}
}
basicModel.setResult(dto);
}
mav.setModel(basicModel);
} catch (Exception e) {
e.printStackTrace();
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 五级地址标准化
*
* @param model
* @return
*/
public Object fiveLevelStructureAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
params.put("refCity", StringUtils.isNotBlank(model.getRefCity()) ? model.getRefCity() : null);
params.put("address", model.getAddress());
// 默认不输出省市区的代码
params.put("showCode", StringUtils.isNotBlank(model.getShowCode()) ? model.getShowCode() : "0");
params.put("complete", "true");
params.put("addressAssessment", "true");
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v3/structure", params, "UTF-8");
} catch (Exception e) {
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/v3/structure", params, "UTF-8");
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("msgCode");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
/**
* 涉农识别-新规则
*
* @param model
* @return
*/
public Object villageIdentifyAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress())) {
mav.setModel(BasicModel.buildBadRequestModel("地址不能为空"));
return mav;
}
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : null);
params.put("address", model.getAddress());
params.put("type", StringUtils.isNotBlank(model.getType()) ? model.getType() : null);
params.put("showFullValue", StringUtils.isNotBlank(model.getShowFullValue()) ? model.getShowFullValue() : null);
params.put("showTestValue", StringUtils.isNotBlank(model.getShowTestValue()) ? model.getShowTestValue() : null);
Integer radius = model.getRadius();
params.put("radius", (radius != null && radius != 0) ? radius.toString() : null);
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
} catch (Exception e) {
try {
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/villageIdentify", params, "UTF-8");
} catch (Exception ex) {
ex.printStackTrace();
}
}
if (json != null) {
int code = json.getInt("msgCode");
String msg = json.getString("msg");
BasicModel basicModel = new BasicModel(code, msg);
if (json.containsKey("result")) {
basicModel.setResult(json.get("result"));
}
mav.setModel(basicModel);
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
public void setDistrictStatService(DistrictStatService districtStatService) {
this.districtStatService = districtStatService;
}
public void setDistrictService(DistrictService districtService) {
this.districtService = districtService;
}
}
File diff suppressed because it is too large Load Diff
@@ -7,9 +7,12 @@ import javax.servlet.http.HttpServletRequest;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.DateUtil;
import com.fangjia.core.util.HttpRequest;
import com.fangjia.dal.mongo.model.*;
import com.fangjia.open.dto.addressGovernance.MatchedBuildingDto;
import com.fangjia.open.model.AvgPriceQueryModel;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.service.datastore.RegionStatService;
import com.fangjia.service.district.DistrictFacilitysService;
import com.fangjia.service.district.DistrictStatRentService;
@@ -1164,7 +1167,41 @@ public class DistrictController {
}
return mav;
}
public Object addrFacilitysAction(DistrictSearchModel model, @Param(name = "city") String city, @Param(name = "address") String address) throws Exception{
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
if (StringUtils.isBlank(address)) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.ARGUMENT_LOSE, "地址必须输入"));
return mav;
} else {
String cat = "district";
HttpRequest request = new HttpRequest();
JSONObject json = null;
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("address", address);
params.put("category", "property");
//新匹配接口
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaNlpService/match", params, "UTF-8");
District property = null;
if (json != null && json.size() > 0) {
MatchedBuildingDto dto = CommonDtoUtils.transformV(MatchedBuildingDto.class, json);
property = districtService.findByNameAndAlias(dto.getCity(), dto.getDistrict(), dto.getName(), cat);
if (null == property) {
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在"));
return mav;
}
DistrictFacilitys facilitys = districtFacilitysService.findByDistrict(property);
mav.setModel(BasicModel.buildOKModel().setResult(facilitys));
}else{
mav.setModel(new HttpCode(OpenHttpCodeEnum.TARGET_DATA_NOT_EXIST, "小区数据不存在"));
}
}
return mav;
}
/**
@@ -20,14 +20,18 @@ import com.fangjia.open.FangjiaApiUtil;
import com.fangjia.open.HttpCode;
import com.fangjia.open.OpenHttpCodeEnum;
import com.fangjia.open.dto.HouseDto;
import com.fangjia.open.dto.UniversalDto;
import com.fangjia.open.dto.addressGovernance.MatchedBuildingDto;
import com.fangjia.open.interceptors.Authority;
import com.fangjia.open.interceptors.Role;
import com.fangjia.open.model.BasicModel;
import com.fangjia.open.model.HousePublishModel;
import com.fangjia.open.model.HouseSearchModel;
import com.fangjia.open.model.RestfulModelAndView;
import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.open.util.FileDao;
import com.fangjia.open.ws.query.CollateralSampleQuery;
import com.fangjia.open.ws.query.SampleQuery;
import com.fangjia.open.ws.query.ShopCollateralSampleQuery;
import com.fangjia.service.address.AddressParser;
import com.fangjia.service.bo.Collateral;
@@ -55,6 +59,7 @@ import com.fangjia.service.open.HousePojo;
import com.fangjia.service.sql.ApiUserService;
import com.fangjia.service.sql.UserService;
import com.fangjia.service.web.MenuService;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.bson.types.ObjectId;
import org.sothis.core.util.Closure;
@@ -65,7 +70,6 @@ import java.io.IOException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@Authority(role = Role.USER)
public class HouseController {
@@ -322,6 +326,117 @@ public class HouseController {
return queryBeans;
}
private List<CollateralSampleQueryBean> convertSampileQueryToBean(SampleQuery... queries) throws Exception{
List<CollateralSampleQueryBean> queryBeans = new ArrayList<CollateralSampleQueryBean>(queries.length);
for(SampleQuery q : queries){
CollateralSampleQueryBean queryBean = new CollateralSampleQueryBean();
BeanUtils.copyProperties(q, queryBean);
// String propertyTypeCode = queryBean.getPropertyTypeCode();
// if(!Arrays.asList("01", "02", "03").contains(propertyTypeCode)){
// throw new Exception("propertyTypeCode取值非法");
// }
String cat = "district";
if(StringUtils.isNotBlank(queryBean.getCity()) && StringUtils.isNotBlank(queryBean.getDistrict()) && StringUtils.isNotBlank(queryBean.getBuildingName())){
District property = districtService.findByNameAndAlias(queryBean.getCity(), queryBean.getDistrict(), queryBean.getBuildingName(), cat);
if(property == null){
throw new Exception("未找到该小区");
}
queryBean.setProperty(property);
}else if(StringUtils.isNotBlank(queryBean.getAddress())){
HttpRequest request = new HttpRequest();
JSONObject json = null;
String address = queryBean.getAddress().replace(" ","").replace("%","");
Map<String, String> params = new HashMap<String, String>();
params.put("city", queryBean.getCity());
params.put("address", address);
params.put("category", "property");
//新匹配接口
json = request.getJsonContent("http://192.168.0.100:32080/FangJiaNlpService/match", params, "UTF-8");
if (json != null && json.size()>0) {
MatchedBuildingDto dto = CommonDtoUtils.transformV(MatchedBuildingDto.class, json);
District property = districtService.findByNameAndAlias(dto.getCity(), dto.getDistrict(), dto.getName(), cat);
if(property == null){
throw new Exception("未找到该小区");
}
queryBean.setProperty(property);
}else{
throw new Exception("未找到该小区");
}
}
String estateType = queryBean.getProperty().getEstateType2();
queryBean.setPropertyTypeCode("01");
if(StringUtils.isNotBlank(estateType)){
if(estateType.equals("普通住宅")){
queryBean.setPropertyTypeCode("01");
}else if(estateType.equals("别墅")){
queryBean.setPropertyTypeCode("02");
}else if(estateType.equals("商住")){
queryBean.setPropertyTypeCode("03");
}
}
Integer decrCondCode = queryBean.getDecrCondCode();
if(decrCondCode == null){
decrCondCode = 2;
}
if(decrCondCode > 5 || decrCondCode < 1){
throw new Exception("decrCondCode取值非法");
}
Integer towardDirectCode = queryBean.getTowardDirectCode();
if(towardDirectCode == null){
towardDirectCode = 3;
}
if(towardDirectCode > 10 || towardDirectCode < 1){
throw new Exception("towardDirectCode取值非法");
}
queryBeans.add(queryBean);
}
return queryBeans;
}
// private List<CollateralSampleQueryBean> convertSampileQueryToBean(SampleQuery... queries) throws Exception{
//
// List<CollateralSampleQueryBean> queryBeans = new ArrayList<CollateralSampleQueryBean>(queries.length);
// for(SampleQuery q : queries){
//
// CollateralSampleQueryBean queryBean = new CollateralSampleQueryBean();
// BeanUtils.copyProperties(q, queryBean);
//
//
// String propertyTypeCode = queryBean.getPropertyTypeCode();
// if(!Arrays.asList("01", "02", "03").contains(propertyTypeCode)){
// throw new Exception("propertyTypeCode取值非法");
// }
// String cat = "district";
// District property = districtService.findByNameAndAlias(queryBean.getCity(), queryBean.getDistrict(), queryBean.getBuildingName(), cat);
// if(property == null){
// throw new Exception("未找到该小区");
// }
// queryBean.setProperty(property);
// Integer decrCondCode = queryBean.getDecrCondCode();
// if(decrCondCode == null){
// decrCondCode = 2;
// }
// if(decrCondCode > 5 || decrCondCode < 1){
// throw new Exception("decrCondCode取值非法");
// }
// Integer towardDirectCode = queryBean.getTowardDirectCode();
// if(towardDirectCode == null){
// towardDirectCode = 3;
// }
// if(towardDirectCode > 10 || towardDirectCode < 1){
// throw new Exception("towardDirectCode取值非法");
// }
// queryBeans.add(queryBean);
// }
// return queryBeans;
// }
public Object detailAction(HousePublishModel model){
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
@@ -774,6 +889,54 @@ public class HouseController {
return mav;
}
public Object caseAction(@Valid SampleQuery sampleQuery){
RestfulModelAndView mav = new RestfulModelAndView();
List<CollateralSampleQueryBean> querys = null;
try{
querys = convertSampileQueryToBean(sampleQuery);
}catch (Exception e){
mav.setModel(BasicModel.buildBadRequestModel(e.getMessage()));
return mav;
}
if(null == sampleQuery.getMax()){
sampleQuery.setMax(10);
}
long start = System.currentTimeMillis();
List<Collateral> collaterals = czbankSampleService.findSample2(querys, sampleQuery.getMax());
long end = System.currentTimeMillis();
System.out.println("用时:"+(end-start));
Collateral collateral = collaterals.get(0);
List<Sample> samples = collateral.getSample();
UniversalDto dto = UniversalDto.newDto();
if(null != collateral.getMaxPrice()){
dto.append("maxUnitPrice", collateral.getMaxPrice());
}
if(null != collateral.getMinPrice()){
dto.append("minUnitPrice", collateral.getMinPrice());
}
dto.append("datas", samples);
// if(samples!= null && samples.size()>=2){
// Sample minSample = samples.get(samples.size()-2);
// Sample maxSample = samples.get(samples.size()-1);
// dto.append("minUnitPrice", minSample.getUnitPrice());
// dto.append("maxUnitPrice", maxSample.getUnitPrice());
// }
// if(samples.size()>2){
// dto.append("datas", samples.subList(0, samples.size()-2));
// }else{
// dto.append("datas", samples);
// }
mav.setModel(BasicModel.buildOKModel().setResult(dto));
return mav;
}
private List<HousePojo> fillDistrictInfoForHouse(FacetQueryResult<com.fangjia.service.bo.HouseBo> houseList, Date endDate,
District district, boolean showDetail, boolean matchModel) {
FacetQueryResult<HousePojo> list = new FacetQueryResult<HousePojo>();
@@ -37,6 +37,8 @@ import com.fangjia.dal.mongo.open.model.EvaluateRatio;
import com.fangjia.dal.seawater.dao.SeawaterDao;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.open.dto.*;
import com.fangjia.open.dto.addressGovernance.StructureDto;
import com.fangjia.open.util.CommonDtoUtils;
import com.fangjia.service.district.*;
import com.fangjia.service.mongo.fangjia.*;
import com.fangjia.service.open.*;
@@ -878,8 +880,13 @@ public class PropertyController {
districtInfoDto.setAddressAlias(district.getAddrAlias());
if (district.getLat2() != null && district.getLat2() != 0 && district.getLng2() != null
&& district.getLng2() != 0) {
districtInfoDto.setLat(district.getLat2() / 10000000000D);
districtInfoDto.setLng(district.getLng2() / 10000000000D);
districtInfoDto.setLat2(district.getLat2() / 10000000000D);
districtInfoDto.setLng2(district.getLng2() / 10000000000D);
}
if (district.getLat() != null && district.getLat() != 0 && district.getLng() != null
&& district.getLng() != 0) {
districtInfoDto.setLat(district.getLat() / 10000000000D);
districtInfoDto.setLng(district.getLng() / 10000000000D);
}
districtInfoDto.setPropertyType(district.getEstateType2());
if (HouseValidator.validateBuildingType(district.getBuildType())) {
@@ -1085,6 +1092,21 @@ public class PropertyController {
return mav;
}
if(model.getFloor() == null){
JSONObject structureOb = addressMatchProxyService.addressStructure(district.getCity(), model.getAddress());
if(structureOb != null){
if(structureOb.containsKey("buildingNumber")){
model.setBuildingNumber(structureOb.getString("buildingNumber"));
}
if(structureOb.containsKey("roomNumber")){
String roomNumber = structureOb.getString("roomNumber");
model.setFloor(getFloor(roomNumber));
}
}
}
if(model.getMaxFloor() == null && model.getFloor() != null &&StringUtils.isNotBlank(model.getBuildingNumber())){
Loudong loudong = loudongService.findForEvaluate(district.getId(), model.getBuildingNumber(), model.getUnitNumber());
if(loudong != null && loudong.getFloors() >= model.getFloor()){
@@ -1108,6 +1130,8 @@ public class PropertyController {
} else {
model.setCalDate(DateUtil.getFirstDayOfWeek(new Date()));
}
Evaluation evaluation = evaluationService.evaluate(district, buildEvaluation(model, request),
model.getCalDate());
Double cityPriceCredit = null;
@@ -1549,6 +1573,9 @@ public class PropertyController {
model.setDistrict(fangjiaOfficialAreaMap.getFangjiaDistrict());
}
}
boolean villaFlag = DataUtil.isVilla(model.getSize(), model.getName(), model.getHeight(), model.getRoom());
District district = null;
Map<String, Object> params = new HashMap<String, Object>();
params.put("city", model.getCity());
@@ -1566,7 +1593,14 @@ public class PropertyController {
if(json.getInt("code")==200){
Map<String, Object> map = (Map<String, Object>) json.get("result");
if((Integer)map.get("precise")>=0){
district = districtService.findByNameAndAlias((String) map.get("city"), (String) map.get("district"), (String) map.get("name"), "district");
String propertyType = (String) map.get("propertyType");
if(villaFlag && StringUtils.isNotBlank(propertyType) && !propertyType.equals("别墅")){
district = districtService.findByNameAndAlias((String) map.get("city"), (String) map.get("district"), (String) map.get("name")+"别墅", "district");
}
if(null == district){
district = districtService.findByNameAndAlias((String) map.get("city"), (String) map.get("district"), (String) map.get("name"), "district");
}
}
}
}
@@ -2773,7 +2807,7 @@ public class PropertyController {
evaluation.setOtherFactor(model.getOtherFactor());
}
evaluation.setRegion(model.getDistrict());
if (0 != model.getRoom()) {
if (null != model.getRoom()) {
evaluation.setRoom(model.getRoom());
}
if (StringUtils.isNotBlank(model.getRoomNumber())) {
@@ -3479,7 +3513,8 @@ public class PropertyController {
if (model.getHall() != null) {
params.add(new QueryParam("hall", model.getHall()));
}
params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l));
// 18年之后抓住的数据没有score字段
// params.add(new QueryParam("score", QueryParam.GREATER_THAN_OPERATOR, 200000000000l));
List<DealPrice> dealPrices = dealPriceService.findDealPrices(params, max);
List<DealPrice> distinctDealPrices = new ArrayList<DealPrice>();
for (DealPrice dealPrice : dealPrices) {
@@ -4057,6 +4092,19 @@ public class PropertyController {
return "fangjia.com";
}
private Integer getFloor(String roomNumber){
if(StringUtils.isNotBlank(roomNumber) && DataUtil.isNumber(roomNumber)){
if(roomNumber.length() == 3){
return Integer.valueOf(roomNumber.substring(0,1));
}else if(roomNumber.length() == 4){
return Integer.valueOf(roomNumber.substring(0,2));
}else{
return null;
}
}
return null;
}
private boolean checkExists(FacetQueryResult<NewHousePojo> houseBoList, NewHousePojo house) {
for (NewHousePojo bo : houseBoList) {
boolean sameName = bo.getName().equals(house.getName());
@@ -5517,7 +5565,7 @@ public class PropertyController {
//ÖܱßÏàËÆÐ¡Çø×â½ð
Integer room = null;
if(model.getRoom()!=0){
if(model.getRoom()!=null && model.getRoom()>0){
room = model.getRoom();
}else{
room = houseService.getDistrictRoomType((float)model.getSize());
@@ -542,7 +542,7 @@ public class UserController {
String code = usersInfoService.randomCode();
System.out.println(code);
try {
smsService.send(model.getMobile(), "【房价网】你的验证码是" + code + ",请立即验证。");
smsService.send(model.getMobile(), "【百盛评估】你的验证码是" + code + ",请立即验证。");
resultModel.setResult(Message.RESULT_SUCCESS);
CookieUtils.setCookie(mobile + "_" + type, code, "/", 60 * 60 * 24 *7, resp);
CacheUtils.putValue(CachePolicyEnum.TINY, FjtUsers.class, mobile, new Date().getTime());
@@ -1285,15 +1285,17 @@ public class WeiXinEvaluationController {
Point point = null;
if(district.getLng2()!=null){
point = new Point(district.getLng2()/10000000000.0, district.getLat2()/10000000000.0);
//boolean flag = InsidePolygon.aboutZone(point, points);
//boolean flag = InsidePolygon.isPtInPoly(point.getLon(), point.getLat(), points);
boolean flag = InsidePolygon2.checkIsInPolygon(point, points);
if(flag){
pbObj = pb;
System.out.println(district.getRegion()+" "+district.getName()+" "+pb.getName());
break;
}
}
//boolean flag = InsidePolygon.aboutZone(point, points);
//boolean flag = InsidePolygon.isPtInPoly(point.getLon(), point.getLat(), points);
boolean flag = InsidePolygon2.checkIsInPolygon(point, points);
if(flag){
pbObj = pb;
System.out.println(district.getRegion()+" "+district.getName()+" "+pb.getName());
break;
}
}
Integer yearAge = getBuildYear(district.getFinishBuildingDate());
if(pbObj!=null){
@@ -7,7 +7,8 @@ import com.fangjia.dal.mongo.model.CityNewPrice;
import com.fangjia.dal.mongo.model.RegionSellStat;
@SuppressWarnings("serial")
public class CityNewPriceDto implements Serializable{
public class
CityNewPriceDto implements Serializable{
/**
*
@@ -26,8 +26,14 @@ public class DistrictInfoDto {
private List<String> alias;
private List<String> addressAlias;
private String videoUrl;
//高德坐标
//维度
private Double lat;
//经度
private Double lng;
// 百度坐标
private Double lat2;
private Double lng2;
private String developer;
private String buildType;
private String estateCompany;
@@ -102,7 +108,22 @@ public class DistrictInfoDto {
private String township;
private String landProperty;
public Double getLat2() {
return lat2;
}
public void setLat2(Double lat2) {
this.lat2 = lat2;
}
public Double getLng2() {
return lng2;
}
public void setLng2(Double lng2) {
this.lng2 = lng2;
}
public String getDistrict() {
return district;
}
@@ -0,0 +1,57 @@
package com.fangjia.open.dto;
import com.fangjia.dal.mongo.model.DistrictFacilitys;
import com.fangjia.open.dto.addressGovernance.MatchedBuildingDto;
import com.fangjia.open.model.RestfulModelAndView;
import java.util.Map;
/**
* @author 13607
* @since 2023/12/12 10:47
*/
public class HouseValueInformationDto {
// 原小区均价返回值
private MatchedBuildingDto communityAvgPrice;
// 原房屋价值评估返回值
private Map<String, Object> addressValue;
//原房源样本返回值
private UniversalDto houseCase;
//原周边POI返回值
private DistrictFacilitys surroundingPoi;
public MatchedBuildingDto getCommunityAvgPrice() {
return communityAvgPrice;
}
public void setCommunityAvgPrice(MatchedBuildingDto communityAvgPrice) {
this.communityAvgPrice = communityAvgPrice;
}
public Map<String, Object> getAddressValue() {
return addressValue;
}
public void setAddressValue(Map<String, Object> addressValue) {
this.addressValue = addressValue;
}
public UniversalDto getHouseCase() {
return houseCase;
}
public void setHouseCase(UniversalDto houseCase) {
this.houseCase = houseCase;
}
public DistrictFacilitys getSurroundingPoi() {
return surroundingPoi;
}
public void setSurroundingPoi(DistrictFacilitys surroundingPoi) {
this.surroundingPoi = surroundingPoi;
}
}
@@ -46,6 +46,11 @@ public class MatchedBuildingDto {
*/
private String propertyType;
/**
* ¾ù¼Û
*/
private Integer avgPrice;
public String getId() {
return id;
}
@@ -125,4 +130,12 @@ public class MatchedBuildingDto {
public void setPropertyType(String propertyType) {
this.propertyType = propertyType;
}
public Integer getAvgPrice() {
return avgPrice;
}
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
}
@@ -9,6 +9,7 @@ import org.hibernate.validator.constraints.NotBlank;
public class AddesssMatchModel extends FangjiaApiModel {
private String city;
private String refCity;
@NotBlank
private String address;
private String suggestDistrict;
@@ -24,6 +25,14 @@ public class AddesssMatchModel extends FangjiaApiModel {
private String showCode;
public String getRefCity() {
return refCity;
}
public void setRefCity(String refCity) {
this.refCity = refCity;
}
public String getAddress_1() {
return address_1;
}
@@ -39,7 +39,13 @@ public class AddressModel extends FangjiaApiModel {
private String dataLevel;
private String keyword;
//是否使用地图poi数据
private boolean useMap;
//涉农到区之后,是否显式后面的字段值
private String showFullValue;
//是否显示涉农测试用字段值
private String showTestValue;
public boolean isContainsLocation() {
return containsLocation;
}
@@ -248,4 +254,28 @@ public class AddressModel extends FangjiaApiModel {
public void setRegion(String region) {
this.region = region;
}
public boolean isUseMap() {
return useMap;
}
public void setUseMap(boolean useMap) {
this.useMap = useMap;
}
public String getShowFullValue() {
return showFullValue;
}
public void setShowFullValue(String showFullValue) {
this.showFullValue = showFullValue;
}
public String getShowTestValue() {
return showTestValue;
}
public void setShowTestValue(String showTestValue) {
this.showTestValue = showTestValue;
}
}
@@ -0,0 +1,941 @@
package com.fangjia.open.model;
import java.util.Date;
/**
* @author 13607
* @since 2023/12/12 14:27
*/
public class HouseInfomationModel extends FangjiaApiModel {
private String city;
private String address;
private String category;
private boolean useMap;
/**
* 判断是不是有房估价
*/
private boolean isYouFang;
/**
* 判断是不是友达
*/
private boolean isYouDa;
/**
* 区域
*/
private String region;
/**
* 区域(新版本估价)
**/
private String district;
/**
* 小区名
*/
private String districtName;
/**
* 小区名(新版本)
*/
private String name;
/**
* 楼号
*/
private String unitNumber;
/**
* 新楼号
*/
private String houseNumber;
private String id;
/**
* 室号
*/
private String roomNumber;
/**
* 面积
*/
private double size;
/**
* 所在楼层
*/
private Integer floor;
/**
* 总楼层
*/
private Integer height;
/**
* 室数
*/
private Integer room;
/**
* 厅数
*/
private int hall;
/**
* 卫数
*/
private int toilet;
/**
* 估价时间
*/
private Date calDate;
private String propertyType;
/**
* 建筑类型
*/
private String buildType;
/**
* 朝向
*/
private String direction;
/**
* 装修情况
*/
private String fitment;
/**
* 装修价格
*/
private int fitmentCharge;
/**
* 景观情况
*/
private String view;
/**
* 噪音情况
*/
private String noise;
/**
* 花园面积
*/
private Double gardenArea;
/**
* 地下室面积
*/
private Double basementArea;
/**
* 厌恶因素
*/
private String[] worryFactor;
/**
* 房龄
*/
private int age;
/**
* 自有车位
*/
private String carport;
/**
* 小区配套
*/
private String facilities;
/**
* 小区位置
*/
private String insidepos;
/**
* 装修年限
*/
private int fitmentAge;
/**
* 采光通风
*/
private String lightingVentilation;
/**
* 边套装置
*/
private String sideset;
/**
* 建筑质量
*/
private String[] buildQuality;
/**
* 厌恶设施
*/
private String worryFacility;
/**
* 产权属性
*/
private String houseProperty;
/**
* 其他因素
*/
private String otherFactor;
/**
* 门面开间
*/
private String facade;
/**
* 公共交通
*/
private String traffice;
/**
* 形状
*/
private String shape;
/**
* 障碍物
*/
private String obstacle;
/**
* 经营性质
*/
private String industry;
/**
* 配套设备
*/
private String supportDevice;
/**
* 距离扶梯位置
*/
private String leftPosition;
/**
* 商铺类型
*/
private String shopType;
/**
* 商铺唯一性
*/
private String shopUnique;
/**
* 商铺位置
*/
private String location;
private String kind;
private boolean isNewDistrict;
private Integer carportPrice;
private String keyword;
private String buildingNumber;
private Integer maxFloor;
private String decoration;
private Integer decorationCost;
private Double gardenSize;
private Double basementSize;
private Double outerSize;
private Double atticSize;
private String negativeFactor;
private String insidePosition;
private int decorationAge;
private boolean convert;
private String estateType;
private String periodType;
private Integer periodCount;
private Integer houseType;
private String clientNo;
private int scope;
private Integer maxPrice;
private Integer minPrice;
private Integer radius;
private Integer limit;
private String app;
private String user;
private Integer avgPrice;
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public String getCategory() {
return category;
}
public void setCategory(String category) {
this.category = category;
}
public boolean isUseMap() {
return useMap;
}
public void setUseMap(boolean useMap) {
this.useMap = useMap;
}
public boolean isYouFang() {
return isYouFang;
}
public void setYouFang(boolean youFang) {
isYouFang = youFang;
}
public boolean isYouDa() {
return isYouDa;
}
public void setYouDa(boolean youDa) {
isYouDa = youDa;
}
public String getRegion() {
return region;
}
public void setRegion(String region) {
this.region = region;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
public String getDistrictName() {
return districtName;
}
public void setDistrictName(String districtName) {
this.districtName = districtName;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getUnitNumber() {
return unitNumber;
}
public void setUnitNumber(String unitNumber) {
this.unitNumber = unitNumber;
}
public String getHouseNumber() {
return houseNumber;
}
public void setHouseNumber(String houseNumber) {
this.houseNumber = houseNumber;
}
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getRoomNumber() {
return roomNumber;
}
public void setRoomNumber(String roomNumber) {
this.roomNumber = roomNumber;
}
public double getSize() {
return size;
}
public void setSize(double size) {
this.size = size;
}
public Integer getFloor() {
return floor;
}
public void setFloor(Integer floor) {
this.floor = floor;
}
public Integer getHeight() {
return height;
}
public void setHeight(Integer height) {
this.height = height;
}
public Integer getRoom() {
return room;
}
public void setRoom(Integer room) {
this.room = room;
}
public int getHall() {
return hall;
}
public void setHall(int hall) {
this.hall = hall;
}
public int getToilet() {
return toilet;
}
public void setToilet(int toilet) {
this.toilet = toilet;
}
public Date getCalDate() {
return calDate;
}
public void setCalDate(Date calDate) {
this.calDate = calDate;
}
public String getPropertyType() {
return propertyType;
}
public void setPropertyType(String propertyType) {
this.propertyType = propertyType;
}
public String getBuildType() {
return buildType;
}
public void setBuildType(String buildType) {
this.buildType = buildType;
}
public String getDirection() {
return direction;
}
public void setDirection(String direction) {
this.direction = direction;
}
public String getFitment() {
return fitment;
}
public void setFitment(String fitment) {
this.fitment = fitment;
}
public int getFitmentCharge() {
return fitmentCharge;
}
public void setFitmentCharge(int fitmentCharge) {
this.fitmentCharge = fitmentCharge;
}
public String getView() {
return view;
}
public void setView(String view) {
this.view = view;
}
public String getNoise() {
return noise;
}
public void setNoise(String noise) {
this.noise = noise;
}
public Double getGardenArea() {
return gardenArea;
}
public void setGardenArea(Double gardenArea) {
this.gardenArea = gardenArea;
}
public Double getBasementArea() {
return basementArea;
}
public void setBasementArea(Double basementArea) {
this.basementArea = basementArea;
}
public String[] getWorryFactor() {
return worryFactor;
}
public void setWorryFactor(String[] worryFactor) {
this.worryFactor = worryFactor;
}
public int getAge() {
return age;
}
public void setAge(int age) {
this.age = age;
}
public String getCarport() {
return carport;
}
public void setCarport(String carport) {
this.carport = carport;
}
public String getFacilities() {
return facilities;
}
public void setFacilities(String facilities) {
this.facilities = facilities;
}
public String getInsidepos() {
return insidepos;
}
public void setInsidepos(String insidepos) {
this.insidepos = insidepos;
}
public int getFitmentAge() {
return fitmentAge;
}
public void setFitmentAge(int fitmentAge) {
this.fitmentAge = fitmentAge;
}
public String getLightingVentilation() {
return lightingVentilation;
}
public void setLightingVentilation(String lightingVentilation) {
this.lightingVentilation = lightingVentilation;
}
public String getSideset() {
return sideset;
}
public void setSideset(String sideset) {
this.sideset = sideset;
}
public String[] getBuildQuality() {
return buildQuality;
}
public void setBuildQuality(String[] buildQuality) {
this.buildQuality = buildQuality;
}
public String getWorryFacility() {
return worryFacility;
}
public void setWorryFacility(String worryFacility) {
this.worryFacility = worryFacility;
}
public String getHouseProperty() {
return houseProperty;
}
public void setHouseProperty(String houseProperty) {
this.houseProperty = houseProperty;
}
public String getOtherFactor() {
return otherFactor;
}
public void setOtherFactor(String otherFactor) {
this.otherFactor = otherFactor;
}
public String getFacade() {
return facade;
}
public void setFacade(String facade) {
this.facade = facade;
}
public String getTraffice() {
return traffice;
}
public void setTraffice(String traffice) {
this.traffice = traffice;
}
public String getShape() {
return shape;
}
public void setShape(String shape) {
this.shape = shape;
}
public String getObstacle() {
return obstacle;
}
public void setObstacle(String obstacle) {
this.obstacle = obstacle;
}
public String getIndustry() {
return industry;
}
public void setIndustry(String industry) {
this.industry = industry;
}
public String getSupportDevice() {
return supportDevice;
}
public void setSupportDevice(String supportDevice) {
this.supportDevice = supportDevice;
}
public String getLeftPosition() {
return leftPosition;
}
public void setLeftPosition(String leftPosition) {
this.leftPosition = leftPosition;
}
public String getShopType() {
return shopType;
}
public void setShopType(String shopType) {
this.shopType = shopType;
}
public String getShopUnique() {
return shopUnique;
}
public void setShopUnique(String shopUnique) {
this.shopUnique = shopUnique;
}
public String getLocation() {
return location;
}
public void setLocation(String location) {
this.location = location;
}
public String getKind() {
return kind;
}
public void setKind(String kind) {
this.kind = kind;
}
public boolean isNewDistrict() {
return isNewDistrict;
}
public void setNewDistrict(boolean newDistrict) {
isNewDistrict = newDistrict;
}
public Integer getCarportPrice() {
return carportPrice;
}
public void setCarportPrice(Integer carportPrice) {
this.carportPrice = carportPrice;
}
public String getKeyword() {
return keyword;
}
public void setKeyword(String keyword) {
this.keyword = keyword;
}
public String getBuildingNumber() {
return buildingNumber;
}
public void setBuildingNumber(String buildingNumber) {
this.buildingNumber = buildingNumber;
}
public Integer getMaxFloor() {
return maxFloor;
}
public void setMaxFloor(Integer maxFloor) {
this.maxFloor = maxFloor;
}
public String getDecoration() {
return decoration;
}
public void setDecoration(String decoration) {
this.decoration = decoration;
}
public Integer getDecorationCost() {
return decorationCost;
}
public void setDecorationCost(Integer decorationCost) {
this.decorationCost = decorationCost;
}
public Double getGardenSize() {
return gardenSize;
}
public void setGardenSize(Double gardenSize) {
this.gardenSize = gardenSize;
}
public Double getBasementSize() {
return basementSize;
}
public void setBasementSize(Double basementSize) {
this.basementSize = basementSize;
}
public Double getOuterSize() {
return outerSize;
}
public void setOuterSize(Double outerSize) {
this.outerSize = outerSize;
}
public Double getAtticSize() {
return atticSize;
}
public void setAtticSize(Double atticSize) {
this.atticSize = atticSize;
}
public String getNegativeFactor() {
return negativeFactor;
}
public void setNegativeFactor(String negativeFactor) {
this.negativeFactor = negativeFactor;
}
public String getInsidePosition() {
return insidePosition;
}
public void setInsidePosition(String insidePosition) {
this.insidePosition = insidePosition;
}
public int getDecorationAge() {
return decorationAge;
}
public void setDecorationAge(int decorationAge) {
this.decorationAge = decorationAge;
}
public boolean isConvert() {
return convert;
}
public void setConvert(boolean convert) {
this.convert = convert;
}
public String getEstateType() {
return estateType;
}
public void setEstateType(String estateType) {
this.estateType = estateType;
}
public String getPeriodType() {
return periodType;
}
public void setPeriodType(String periodType) {
this.periodType = periodType;
}
public Integer getPeriodCount() {
return periodCount;
}
public void setPeriodCount(Integer periodCount) {
this.periodCount = periodCount;
}
public Integer getHouseType() {
return houseType;
}
public void setHouseType(Integer houseType) {
this.houseType = houseType;
}
public String getClientNo() {
return clientNo;
}
public void setClientNo(String clientNo) {
this.clientNo = clientNo;
}
public int getScope() {
return scope;
}
public void setScope(int scope) {
this.scope = scope;
}
public Integer getMaxPrice() {
return maxPrice;
}
public void setMaxPrice(Integer maxPrice) {
this.maxPrice = maxPrice;
}
public Integer getMinPrice() {
return minPrice;
}
public void setMinPrice(Integer minPrice) {
this.minPrice = minPrice;
}
public Integer getRadius() {
return radius;
}
public void setRadius(Integer radius) {
this.radius = radius;
}
public Integer getLimit() {
return limit;
}
public void setLimit(Integer limit) {
this.limit = limit;
}
public String getApp() {
return app;
}
public void setApp(String app) {
this.app = app;
}
public String getUser() {
return user;
}
public void setUser(String user) {
this.user = user;
}
public Integer getAvgPrice() {
return avgPrice;
}
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
}
@@ -70,12 +70,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
/**
* 总楼层
*/
private int height;
private Integer height;
/**
* 室数
*/
private int room;
private Integer room;
/**
* 厅数
@@ -345,19 +345,19 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
this.floor = floor;
}
public int getHeight() {
public Integer getHeight() {
return height;
}
public void setHeight(int height) {
public void setHeight(Integer height) {
this.height = height;
}
public int getRoom() {
public Integer getRoom() {
return room;
}
public void setRoom(int room) {
public void setRoom(Integer room) {
this.room = room;
}
@@ -0,0 +1,10 @@
package com.fangjia.open;
public class test {
public static void main(String[] args) {
String a = "123545";
String b ="i980";
System.out.println(a.matches("\\d+"));
System.out.println(b.matches("\\d+"));
}
}
@@ -0,0 +1,181 @@
package com.fangjia.open.ws.query;
import org.hibernate.validator.constraints.NotBlank;
import javax.validation.constraints.Max;
import javax.validation.constraints.Min;
import javax.validation.constraints.NotNull;
import java.util.Date;
public class SampleQuery {
/**城市*/
// @NotBlank
private String city;
/**区域*/
// @NotBlank
private String district;
// @NotBlank
/**小区名*/
private String buildingName;
/**楼栋名称*/
private String buildingNoName;
/**物业类型编码*/
private String propertyTypeCode;
/**是否有电梯 0:无 1:有*/
private Integer isElevator;
/**总楼层*/
private Integer aboveFloorAmount;
/**所在楼层*/
private Integer floorNumber;
/**
* 装修情况编码
* 1:毛坯
* 2:简装(默认)
* 3:精装
* 4:豪装
* */
private Integer decrCondCode;
/**面积**/
private Double strctureArea;
/**
* 朝向情况编码
* 1:西南
* 2:东南
* 3:南(默认)
* 4:西北
* 5:东北
* 6:东
* 7:西
* 8:北
* 9:东西
* 10:南北
* */
private Integer towardDirectCode;
private Date updateDate;
@NotBlank
private String address;
@Min(1)
@Max(10)
private Integer max;
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
public String getBuildingName() {
return buildingName;
}
public void setBuildingName(String buildingName) {
this.buildingName = buildingName;
}
public String getBuildingNoName() {
return buildingNoName;
}
public void setBuildingNoName(String buildingNoName) {
this.buildingNoName = buildingNoName;
}
public String getPropertyTypeCode() {
return propertyTypeCode;
}
public void setPropertyTypeCode(String propertyTypeCode) {
this.propertyTypeCode = propertyTypeCode;
}
public Integer getIsElevator() {
return isElevator;
}
public void setIsElevator(Integer isElevator) {
this.isElevator = isElevator;
}
public Integer getAboveFloorAmount() {
return aboveFloorAmount;
}
public void setAboveFloorAmount(Integer aboveFloorAmount) {
this.aboveFloorAmount = aboveFloorAmount;
}
public Integer getFloorNumber() {
return floorNumber;
}
public void setFloorNumber(Integer floorNumber) {
this.floorNumber = floorNumber;
}
public Double getStrctureArea() {
return strctureArea;
}
public void setStrctureArea(Double strctureArea) {
this.strctureArea = strctureArea;
}
public Integer getDecrCondCode() {
return decrCondCode;
}
public void setDecrCondCode(Integer decrCondCode) {
this.decrCondCode = decrCondCode;
}
public Integer getTowardDirectCode() {
return towardDirectCode;
}
public void setTowardDirectCode(Integer towardDirectCode) {
this.towardDirectCode = towardDirectCode;
}
public Date getUpdateDate() {
return updateDate;
}
public void setUpdateDate(Date updateDate) {
this.updateDate = updateDate;
}
public Integer getMax() {
return max;
}
public void setMax(Integer max) {
this.max = max;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
}
@@ -0,0 +1,18 @@
package com.fangjia;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.codec.binary.Hex;
import java.util.Scanner;
public class test {
public static void main(String[] args) {
while (true){
Scanner scanner = new Scanner(System.in);
String password = scanner.nextLine();
String p =DigestUtils.md5Hex(password);
System.out.println(p);
}
}
}
@@ -109,9 +109,11 @@
</div>
</div>
</div>
<div class="content-block">
<p><a href="../weiXinEvaluation/form?id=${id}" class="button button-big button-fill evaluation_btn external">房产评估</a></p>
</div>
<div class="content-block">
<p>
<a href="../weiXinEvaluation/form?id=${id}" class="button button-big button-fill evaluation_btn external">房产评估</a>
</p>
</div>
</#if>
</div>
</div>
Binary file not shown.

After

Width:  |  Height:  |  Size: 7.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 641 KiB

@@ -21,12 +21,12 @@
</head>
<body>
<!-- 替换百盛图片 -->
<div class="page-group">
<div class="page page-current">
<div class="content login_bg2">
<div class="login_logo" style="margin-top: 0px;"><img src="../static/images/9nf1iubwu6.jpg" style="width: 80%;height: 50px;margin: 30px auto;"></div>
<div class="login_logo" style="margin-top: 0px;"><img src="../static/images/bh1.png" id="bannersDiv" style="width: 100%;height: 220px;"></div>
<div class="login_logo" style="margin-top: 0px;"><img src="../static/images/baisheng_logo.jpg" style="width: 80%;height: 50px;margin: 30px auto;"></div>
<div class="login_logo" style="margin-top: 0px;"><img src="../static/images/bs_bananer.jpg" id="bannersDiv" style="width: 100%;height: 220px;"></div>
<div class="login_content" style="margin-top: 8%;">
<div class="login_input" id="bhFrom">
<div class="inputInfo01">
+23 -1
View File
@@ -530,7 +530,29 @@
</exclusion>
</exclusions>
</dependency>
<!-- <dependency>-->
<!-- <groupId>com.fangjia</groupId>-->
<!-- <artifactId>fangjia-open</artifactId>-->
<!-- <version>0.0.1-SNAPSHOT</version>-->
<!-- <scope>compile</scope>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.aliyun</groupId>-->
<!-- <artifactId>aliyun-java-sdk-address-purification</artifactId>-->
<!-- <version>1.0.1</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.aliyun</groupId>-->
<!-- <artifactId>aliyun-java-sdk-core</artifactId>-->
<!-- <version>4.5.9</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>com.alibaba</groupId>-->
<!-- <artifactId>fastjson</artifactId>-->
<!-- <version>1.2.83</version>-->
<!-- </dependency>-->
</dependencies>
</dependencies>
</project>
@@ -2,36 +2,35 @@ package com.fangjia.script
public enum Watcher {
//离职同事
LiuPeiPei('13621626685', 'duxiaoyu@fangjia.com'),
YanZhou('13621626685','duxiaoyu@fangjia.com'),
HuangXiaoHu('13621626685', 'duxiaoyu@fangjia.com'),
SongYingXia('13621626685', 'duxiaoyu@fangjia.com'),
YuKai('13621626685', 'duxiaoyu@fangjia.com'),
RuanYaDong('13621626685', 'duxiaoyu@fangjia.com'),
Reda('13621626685', 'duxiaoyu@fangjia.com'),
Tomzhao('13621626685','duxiaoyu@fangjia.com'),
YuHai('13621626685','duxiaoyu@fangjia.com'),
Shibb('13621626685','duxiaoyu@fangjia.com'),
HeLvYuan('13621626685','duxiaoyu@fangjia.com'),
WangYuHui('13621626685','duxiaoyu@fangjia.com'),
LinXiaoShuai('15801810393','linxiaoshuai@fangjia.com'),
FangRui('17717680589','fangrui@fangjia.com'),
//DuXiaoYu('13621626685', 'duxiaoyu@fangjia.com'),
LiuPeiPei('13621626685', 'wangwenyan@fangjia.com'),
YanZhou('13621626685','wangwenyan@fangjia.com'),
HuangXiaoHu('13621626685', 'wangwenyan@fangjia.com'),
SongYingXia('13621626685', 'wangwenyan@fangjia.com'),
YuKai('13621626685', 'wangwenyan@fangjia.com'),
RuanYaDong('13621626685', 'wangwenyan@fangjia.com'),
Reda('13621626685', 'wangwenyan@fangjia.com'),
Tomzhao('13621626685','wangwenyan@fangjia.com'),
YuHai('13621626685','wangwenyan@fangjia.com'),
Shibb('13621626685','wangwenyan@fangjia.com'),
HeLvYuan('13621626685','wangwenyan@fangjia.com'),
WangYuHui('13621626685','wangwenyan@fangjia.com'),
LinXiaoShuai('15801810393','wangwenyan@fangjia.com'),
FangRui('17717680589','wangwenyan@fangjia.com'),
DuXiaoYu('13621626685', 'wangwenyan@fangjia.com'),
//在职同事
DuXiaoYu('13916076378', 'wangwenyan@fangjia.com'),
LiXue('13311649646','lixue@fangjia.com'),
LiXue('13311649646','wangwenyan@fangjia.com'),
WangWenYan('13916076378','wangwenyan@fangjia.com'),
GaoJuan('15021384995','gaojuan@fangjia.com'),
Hubingliang('15179856208','hubingliang@126.com'),
ZhouTongYun('13585834921','zhoutongyun@fangjia.com'),
YinJiHuan('18217642009','yinjihuan@fangjia.com'),
Xuhe('13862416422','xuhe@fangjia.com'),
ZongZhiMin('18260046486','zongzhimin@fangjia.com'),
RaoYanHui('18516539752','raoyanhui@fangjia.com'),
BaYi('15300424133','bayi@fangjia.com'),
WeiLei('18101751850','weilei@fangjia.com'),
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
Hubingliang('15179856208','wangwenyan@fangjia.com'),
ZhouTongYun('13585834921','wangwenyan@fangjia.com'),
YinJiHuan('18217642009','wangwenyan@fangjia.com'),
Xuhe('13862416422','wangwenyan@fangjia.com'),
ZongZhiMin('18260046486','wangwenyan@fangjia.com'),
RaoYanHui('18516539752','wangwenyan@fangjia.com'),
BaYi('15300424133','wangwenyan@fangjia.com'),
WeiLei('18101751850','wangwenyan@fangjia.com'),
JiangYaHui('13684913780','wangwenyan@fangjia.com'),
ShenJian('13764369683', 'shenjian@fangjia.com')
private final String mobile;
private final String email;
@@ -124,13 +124,13 @@ public class ManualOperationDataStat extends BaseBatch{
println(content)
sendEmail(Watcher.WangWenYan.email, "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("wangchencen@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("caoyuyan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("shenyuan@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("xuminxia@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("huangcunliang@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("yaojingmeng@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("muliangshu@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
// sendEmail("sunjian@fangjia.com", "ºǫ́ͳ¼ÆÁ¿", content)
}
}
@@ -1,5 +1,7 @@
package com.fangjia.script.ana
import com.fangjia.script.util.ExcelUtils
import java.util.Date;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -40,9 +42,46 @@ public class Test {
// //test.run("district", "上海");
// test.analyDistrictPk("district", "天津");//小区统计与小区的对比
// println new Date(new ObjectId('5469bb4cda29ee73010fa577').getTime())
test.addTestData();
// test.addTestData();
test.findRepeatGuidingPrice()
}
def findRepeatGuidingPrice(){
def q = new BasicDBObject();
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-03-01", "yyyy-MM-dd")))
def dbcursor = GUIDING_PRICE_COLL.find(q)
def map = [:]
for(def d : dbcursor){
def key = d.city +"##"+d.region+"##"+d.name
def rets = map.get(key)
if(!rets){
rets = []
map.put(key, rets)
}
rets << d
}
def datas = []
datas << ["id", "城市","区域","名称","添加人","指导价","日期"]
map.each{key,list->
if(list.size()>1){
def userNames = []
for(def ret : list){
userNames << ret.guiding_person
}
if(userNames.contains("付贵生")){
for(def r : list){
if(r.guiding_person!="付贵生"){
GUIDING_PRICE_COLL.remove(new BasicDBObject("_id", r._id))
}
}
}
}
}
// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas")
}
public void addTestData(){
@@ -28,7 +28,8 @@ import org.bson.types.ObjectId
class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
// public static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
public static final DBCollection SEAWEED_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public final static DBCollection COMMAND_COLL = new MongoClient("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('calc_command')
public final static DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
public final static DBCollection REGION_COLL = MongoUtils.getCollectionByName("fangjia", "region_block", "mongo.db.fangjia.com", 27017)
@@ -215,7 +216,7 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
DBObject query = new BasicDBObject()
query.put("city", city)
query.put("cat", "district")
query.put("visible", 0)
query.put("status", 0)
DBObject fields = new BasicDBObject()
fields.put("city", 1)
@@ -227,11 +228,9 @@ class InsertMonitorSquareRentCalcCommandBatch extends BaseBatch {
int tryTimes = 5
List<BasicDBObject> districts = new ArrayList<BasicDBObject>()
for(int i=0; i < tryTimes; i++) {
DSCursor cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
def cur = SEAWEED_COLL.find(query, fields).batchSize(5000)
int totalCount = cur.count()
Iterator<DBObject> iter = cur.iterator()
while(iter.hasNext()) {
BasicDBObject dbo = (BasicDBObject)iter.next()
for(def dbo : cur) {
districts.add(dbo)
}
if(districts.size() == totalCount) {
@@ -9,7 +9,7 @@ import com.mongodb.DBCollection
/**
* 深圳公众号成交数据录入
*/
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳6月份二手房成交数据汇总.xls", 0, 1)
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1)
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
@@ -58,10 +58,16 @@ for(def d : datas){
}catch(Exception e){
println "room error"
}
map.hall = arrs[1].toString().toInteger()
map.toilet = arrs[3].toString().toInteger()
if(arrs[1].toString()){
map.hall = arrs[1].toString().toInteger()
}
if(arrs.size()==4 && arrs[3].toString()){
map.toilet = arrs[3].toString().toInteger()
}
}else{
map.room = roomStr.toString().toDouble().toInteger()
if(roomStr.toString()){
map.room = roomStr.toString().toDouble().toInteger()
}
}
}
@@ -74,7 +80,7 @@ for(def d : datas){
if(trade_date){
map.trade_date = trade_date
}else{
def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31")
def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30")
map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd")
}
@@ -0,0 +1,77 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
/**
* 添加人工指导价,并重算价格
* 小付的人工指导价不能覆盖
*/
class BatchAddPersonGuiPrice {
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
static main(args) {
def task = new BatchAddPersonGuiPrice()
task.run()
}
def run(){
def path = "D:\\数据\\价格\\常南苏徐无人工指导价录入.xlsx"
def datas = ExcelUtils.read(path, 0, 1)
for(def d : datas){
def row = findGuidingPrice(d['B'], d['C'], d['D'])
if(!row){
def obj = CommonDatautils.findDistrictById(d['A'])
save(d['B'], d['C'], d['D'], d['M'], obj.estate_type2)
//发重算均价命令
addPrice(d['B'], d['C'], d['D'], obj.estate_type2, DateUtils.parseFromTightByPattern('2023-01-01','yyyy-MM-dd'),new Date())
}
}
}
def findGuidingPrice(city, region, name) {
def q = new BasicDBObject();
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("guiding_person" , "付贵生")
q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2022-11-01", "yyyy-MM-dd")))
def dbo = GUIDING_PRICE_COLL.findOne(q)
return dbo
}
def save(city, region, name, price, estate_type){
DBObject obj = new BasicDBObject();
obj.put("city", city);
obj.put("region", region);
obj.put("name", name);
obj.put("guiding_date", new Date()-7);
obj.put("guiding_value", price.toString().toDouble().toInteger());
obj.put("category", "district");
obj.put("guiding_type", 0);
obj.put("c_date", new Date());
obj.put("m_date", new Date());
obj.put("estate_type" , estate_type);
obj.put("house_type" , 0)
obj.put("guiding_person" , "陈杰")
GUIDING_PRICE_COLL.save(obj);
}
def addPrice(city, region, name, estateType2,startDate, endDate){
def weekends = DateUtils.getWeekendBetweenDates(startDate, endDate)
for(def weekend : weekends){
CommonUtils.recalculationAverage(city, region, name, estateType2, weekend,false,0)
}
}
}
@@ -90,11 +90,16 @@ class BigdataCommunityBatch extends BaseBatch{
if(today > 10){
delPgData()
saveToPgSql(rows)
//¸üо²Ì¬Êý¾Ý±í
updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat")
// delPgData()
// saveToPgSql2(datas)
}
def addDeleteDist(dists, ccbMaps, rets){
@@ -311,7 +316,7 @@ class BigdataCommunityBatch extends BaseBatch{
def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
def index = 0
sqlInstance.withBatch(10000, sql){ ps->
sqlInstance.withBatch(50000, sql){ ps->
for(def r : newRows) {
index++
try{
@@ -339,4 +344,12 @@ class BigdataCommunityBatch extends BaseBatch{
return rows;
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
sqlInstance.executeUpdate(sql)
}
}
}
@@ -39,7 +39,7 @@ class BigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
@@ -60,20 +60,14 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
//def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date())
// def citys = CcbCommonUtils.citys;
def citys = CcbCommonUtils.getRunningCitys(args)
if(args && args.size()>=2){
curDate = args[1].toString().toInteger()
}
println "curDate:"+curDate
// def allCitys = CommonDatautils.findAllCity()
// for(def city : allCitys){
//// if(!ccbCitys.contains(city)){
// addDealPriceMatch(city);
//// }
// }
def citys = CcbCommonUtils.getRunningCitys(args)
println "curDate:"+curDate
def allCitys = CommonDatautils.findAllCity()
for(def city : allCitys){
addDealPriceMatch(city);
}
def rows = []
def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
@@ -84,7 +78,7 @@ class BigdataDealBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def ppeDists = [:]
def dealIds = findHistoryDealIds(city)
def dealIds = findHistoryDealIds(CcbCommonUtils.getCcbCity(city))
println "历史id加载量:"+dealIds.size()
def statMap = [:]//getDistrictStatByCity(city, weekend)
if(city=="上海"){
@@ -115,11 +109,29 @@ class BigdataDealBatch extends BaseBatch{
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成 " + rows.size()
// delPgData()
saveToPgSql(rows)
// def datas = readTxt("D:\\数据\\建行\\建行成交\\MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_20230928_0001.dat")
// saveToPgSql2(datas)
}
def readTxt(def fileName){
File file = new File(fileName);
def rows = [];
file.eachLine{
def row = it.split("\\|@\\|")
rows << row
}
return rows;
}
def createData(city, def rets, cityCodeDatas, distCodeMap, source, dealIds){
def result = Collections.synchronizedList(new ArrayList());
dealIds = Collections.synchronizedList(dealIds);
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
@@ -147,12 +159,14 @@ class BigdataDealBatch extends BaseBatch{
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap, ccbPriceMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 友达:"+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}
}else{
def ret = converData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
dealIds << trade._id.toString()
println totalCount.get() +" 第三方:"+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
}
@@ -167,6 +181,7 @@ class BigdataDealBatch extends BaseBatch{
}
println "查询完成......."
latch.await(5, TimeUnit.MINUTES);
return result
}
@@ -217,7 +232,7 @@ class BigdataDealBatch extends BaseBatch{
def price = ccbPriceMap.get(obj._id.toString())
if(price){
def mod = java.lang.Math.abs((trade.price-price)*100/price)
if(mod<=10){
if(mod<=20){
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << trade.city
@@ -515,6 +530,7 @@ class BigdataDealBatch extends BaseBatch{
def getCityData(def city, def statMap, def ppeDistsMap){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
println city +" 小区量:"+dists.size()
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -635,7 +651,7 @@ class BigdataDealBatch extends BaseBatch{
}
def findHistoryDealIds(def city){
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}' "
def sql = "SELECT gujia_bds_mdl_id from base.t0831_bigdata_deal_gj_h where urbn_nm='${city}'"
def ids = []
try{
pgsqlInstance.eachRow(sql,{
@@ -723,6 +739,14 @@ class BigdataDealBatch extends BaseBatch{
println city +" "+trades.size()+" "+new Date().toString()
}
def delPgData(){
def month = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28"
pgsqlInstance.execute("delete from base.t0831_bigdata_deal_gj_h where p9_start_date='${month}'")
println "base.t0831_bigdata_deal_gj_h 数据已删除"
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
@@ -753,4 +777,43 @@ class BigdataDealBatch extends BaseBatch{
println "end: "+new Date()
}
def saveToPgSql2(rows){
def newRows = []
for(def r : rows) {
def newR = []
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
// r[i] = null
newR << null
}else{
newR << r[i]
}
}
newR << null
newR << "2023-09-28"
newR << null
newR << '2999-12-31'
newR << null
newR << null
newR << null
newRows << newR
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : newRows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
}
}
println "end: "+new Date()
}
}
@@ -85,6 +85,7 @@ class BigdataForsaleBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def startDate = lastMonth+"18"
def endDate = curMonth+"19"
println "·¿Ô´Ê±¼ä£º"+startDate+" "+endDate
@@ -196,6 +197,7 @@ class BigdataForsaleBatch extends BaseBatch{
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.room_structure_code = CcbCodeUtils.getRoomStructureCode(m.room_structure)
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
@@ -311,7 +313,7 @@ class BigdataForsaleBatch extends BaseBatch{
data << m.floor==null?"":m.floor
data << m.height==null?"":m.height
data << m.estate_type==null?"":m.estate_type
data << m.house_structure_code==null?"":m.house_structure_code
data << m.room_structure_code==null?"":m.room_structure_code
data << m.build_structure_code==null?"":m.build_structure_code
data << m.area==null?"":m.area
data << ""
@@ -670,6 +672,7 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
@@ -731,7 +734,7 @@ class BigdataForsaleBatch extends BaseBatch{
QueryBuilder date_range_mdate = QueryBuilders.rangeQuery("m_date").from(startMdate.getTime()).to(endMdate.getTime()).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range).must(date_range_mdate)
.must(date_range).must(area_range).must(date_range_mdate)//.must(QueryBuilders.termQuery("root", "lianjia.com"))
if(!alias){
alias = []
@@ -882,6 +885,7 @@ class BigdataForsaleBatch extends BaseBatch{
rowMap.room = seawater.room=="null"?null:seawater.room
rowMap.hall = seawater.hall=="null"?null:seawater.hall
rowMap.toilet = seawater.toilet=="null"?null:seawater.toilet
rowMap.room_structure = seawater.room_structure=="null"?null:seawater.room_structure
if(CcbCommonUtils.rootMap.get(seawater.root)){
rowMap.root = CcbCommonUtils.rootMap.get(seawater.root)
}else{
@@ -0,0 +1,192 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.BaseBatch
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
/***
* 同步上个月城市,区域,小区价格到base库中
*/
class BigdataSynchPrice extends BaseBatch{
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) {
runBatch(new BigdataSynchPrice(), args)
}
@Override
void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
synchCityPrice(citys)
synchDistrictPrice(citys)
synchCommPirce(citys)
}
def synchCityPrice(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_city_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步城市均价 start: "+new Date()
def sql = "insert into base.t0831_city_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(500, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def synchDistrictPrice(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_district_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'"; //
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gxda_admndiv_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_bds_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步区域均价 start: "+new Date()
def sql = "insert into base.t0831_district_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(1000, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
def synchCommPirce(citys){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${lastMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_comm_basic_prc
data << r.gujia_bds_clmtd
data << r.gujiabds_fwd_samp_rto
data << r.gujia_bds_mns_grd
data << r.gujia_bds_cfdrg
data << r.gujiabdsarndcmprccrlt
data << r.gujia_bds_cfdc_dgr
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << null
data << DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")+"-28"
data << null
data << '2999-12-31'
data << null
data << null
data << null
allRows << data
});
}
println "同步小区价格 start: "+new Date()
def sql = "insert into base.t0831_community_base_price_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(50000, sql){ ps->
for(def r : allRows) {
try{
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], r[21], r[22], r[23], r[24], r[25])
ps.toString()
}catch(Exception e){
e.printStackTrace()
}
}
}
println "end: "+new Date()
}
}
@@ -30,6 +30,9 @@ class CcbCommonUtils {
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static final DBCollection GUIDING_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "guiding_price", "mongo.db.fangjia.com")
@@ -37,7 +40,7 @@ class CcbCommonUtils {
public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017)
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42","xflapp.com":"43"]
public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"]
@@ -225,7 +228,7 @@ class CcbCommonUtils {
def lists = []
println dist.name +" "+total
for(def d : dbcursor){
def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
def key=d.city+"##"+d.match_region+"##"+d.match_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
@@ -432,7 +435,7 @@ class CcbCommonUtils {
q.put("cat", cat);
q.put("city", city);
// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13
// q.put("name","½ðµØçôÍå")
// q.put("name","汤堡小区")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -443,7 +446,7 @@ class CcbCommonUtils {
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "[${city}] 原始:"+dbsour.count()
def rets = []
@@ -467,7 +470,7 @@ class CcbCommonUtils {
public static def findAllDistrictByCityAndRegion(def city, def region, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", region)
@@ -482,7 +485,7 @@ class CcbCommonUtils {
}
}
// def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = DBUtils.seaweed.find(q,keys).batchSize(2000)
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
println "原始:"+dbsour.count()
def rets = []
for(def d : dbsour){
@@ -679,4 +682,24 @@ class CcbCommonUtils {
return ids;
}
static def findAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from t_app_p9pha_plda.avm_community_base_price_gj_h where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
}
@@ -29,7 +29,7 @@ class DealPriceDataProcess {
// task.process()//运当成交pe值不好时行,拉升成交价格及删除pe太大数据
// task.addLianJiaPrice();
task.deleteThirdPartyData()//删除第三方
// task.delDealByPe()
// task.delDealByPe()//删除成交
// task.delDealData()
// task.exportThirdParty()
@@ -37,49 +37,49 @@ class DealPriceDataProcess {
}
def delDealByPe(){
def yearMonth = 202201
def thirdMonth = 202112
def yearMonth = 202401
// def rets = findAllDealByPe(yearMonth, 70)
// for(def r : rets){
// deleteDealPrice(r)
def rets = findAllDealByPe(yearMonth, 18)
for(def r : rets){
deleteDealPrice(r)
}
// def list = findAllThirdByPe(yearMonth, 70)
// for(def r : list){
// deletThirdPrice(r, thirdMonth)
// }
// def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
// for(def d : datas){
//// if(d['G']){
//// def rets = findDealByPe(d['A'], yearMonth, d['G'])
//// for(def r : rets){
//// deleteDealPrice(r)
//// }
//// }
//
//// if(d['H']){
//// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
//// for(def m : rets){
//// def precent = d['H']/100.0
//// def newPrice = null
//// if(m.unit_price > m.deal_price){
//// newPrice = m.deal_price*precent + m.deal_price
//// }else if(m.unit_price < m.deal_price){
//// newPrice = m.deal_price - (m.deal_price*precent)
//// }
//// if(newPrice){
//// updateDealPrice(m, newPrice.toInteger())
//// }
//// }
//// }
//
//// if(d['M']){
//// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
//// for(def r : rets){
//// deletThirdPrice(r, thirdMonth)
//// }
//// }
// }
def list = findAllThirdByPe(yearMonth, 70)
for(def r : list){
deletThirdPrice(r, thirdMonth)
}
def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1)
for(def d : datas){
// if(d['G']){
// def rets = findDealByPe(d['A'], yearMonth, d['G'])
// for(def r : rets){
// deleteDealPrice(r)
// }
// }
// if(d['H']){
// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20)
// for(def m : rets){
// def precent = d['H']/100.0
// def newPrice = null
// if(m.unit_price > m.deal_price){
// newPrice = m.deal_price*precent + m.deal_price
// }else if(m.unit_price < m.deal_price){
// newPrice = m.deal_price - (m.deal_price*precent)
// }
// if(newPrice){
// updateDealPrice(m, newPrice.toInteger())
// }
// }
// }
// if(d['M']){
// def rets = findThirdByPe(d['A'], yearMonth, d['M'])
// for(def r : rets){
// deletThirdPrice(r, thirdMonth)
// }
// }
}
}
@@ -151,9 +151,10 @@ class DealPriceDataProcess {
}
def deleteDealPrice(r){
def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
"and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
sqlInstance236.executeUpdate(sql)
// def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " +
// "and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')"
def sql = "delete from base.t0831_bigdata_deal_gj_h where gujia_comm_id='${r.commid}' and txn_dt='${r.dealdate}' and guiabconstructionarea=${r.buildarea} and gujia_bds_trdtckt_prc=${r.deal_price}"
pgsqlInstance.executeUpdate(sql)
}
def deletThirdPrice(r, thirdMonth){
@@ -177,19 +178,14 @@ class DealPriceDataProcess {
}
def findAllDealByPe(yearMonth, peVal){
def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='成交' and pe>${peVal}";
def sql = "select gujia_comm_id as commid,gujiabds_valt_unitprc as unit_price,gujiabds_to_mkprc_mny as target_avg_price,gujia_area as buildarea,to_char(mdl_dt,'yyyy-mm-dd') as dealdate from t_app_p9pha_plda.avm_evaluation_item_gj_h where urbn_nm in ('昆明','南京','苏州','重庆') and gujiabds_clc_yrmo_val=${yearMonth} and gujia_data_tp='成交' and gujia_mns_rto>${peVal} ";
def rets = []
sqlInstance236.eachRow(sql,{r->
pgsqlInstance.eachRow(sql,{r->
def map = [:]
map.cityname = r.cityname
map.cityid=r.cityid
map.distid = r.distid
map.commid = r.commid
map.commname = r.commname
map.deal_price = r.target_avg_price
map.unit_price = r.unit_price
map.buildarea = r.buildarea
map.ppe = r.pe
map.dealdate = r.dealdate
rets << map
})
@@ -392,12 +388,12 @@ class DealPriceDataProcess {
def deleteThirdPartyData(){
def citys = CcbCommonUtils.citys_all
def month = 202306
def month = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//上个月日期
def rows = []
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def set = new HashSet()
for(def city : citys) {
def rets = findPeGteThirdParty2(city, 202307)
def rets = findPeGteThirdParty2(city, DateUtils.formatDateByPattern(new Date(),"yyyyMM"))//当前月日期
def total = 0
println city +" "+rets.size()
if(rets){
@@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
//添加自己库中的人工指导价
def prices = CcbCommonUtils.findPersonPrice(city, "2023-06-26", "2023-07-26")
def prices = CcbCommonUtils.findPersonPrice(city, "2024-01-10", "2024-01-28")
// def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22")
if(prices){
def distMap = getDistrictMap(city)
@@ -84,8 +84,7 @@ class GuiDingPriceBatch extends BaseBatch{
saveToPgSql(rows)
//
//¸üо²Ì¬Êý¾Ý±í
// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28")
}
@@ -244,14 +243,7 @@ class GuiDingPriceBatch extends BaseBatch{
println "end: "+new Date()
}
def updateVersion(def time){
def tables = ["t0831_website_config_gj_h","t0831_bigdata_district_gj_h","t0831_evaluate_rate_gj_h","t0831_bigdata_poi_gj_h","t0831_guided_price_gj_h","t0831_community_staging_gj_h","t0831_community_guide_price_gj_h"]
for(def tableName : tables){
def sql = "update base."+tableName+" set p9_start_date='"+time+"'"
pgsqlInstance.executeUpdate(sql)
}
}
@@ -2,6 +2,7 @@ package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.RandomUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
@@ -1287,10 +1288,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx"
def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20240127.xlsx"
//ExcelUtils.multiThreadingModify(fileName, {d->
// if(d['A']){
ExcelUtils.multiThreadingModify(fileName, {d->
if(d['A']){
// def region = RegionData.formatRegion(d['B'], d['C'])
// if(region){
// def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district")
@@ -1299,37 +1300,42 @@ def fileName = "D:\\
// d['H'] = dist._id.toString()
// }
// }
//// if(!d['F']){
//// def dist = CommonDatautils.findDistrictById(d['A'])
//////
//// if(dist){
////// d['H'] = dist.region
////// d['G'] = dist.name
//// d['F'] = dist.status
//// }
//// }
//
// }
//},0)
if(!d['I']){
def dist = CommonDatautils.findDistrictById(d['C'])
if(dist){
d['F'] = dist.region
// d['H'] = dist.name
d['G'] = dist.status
// if(d['A']=="天门" || d['A']=="迪庆"){
// def val = d['E'].toString().toDouble().toInteger() + RandomUtil.random(-200,200)
// d['K'] = val.toInteger()
// }
}
}
}
},0)
//def fileName = "C:\\Users\\gaojuan\\Desktop\\房价网新增小区数据_0427.xlsx"
//def fileName = "D:\\数据\\建行\\建行成交\\咸阳成交.xlsx"
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
// if(count && count==1){
// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
// d['E'] = dist.city
//// def count = CommonDatautils.findDistrictCount(null, d['B'], d['C'], "district")
//// if(count && count==1){
//// def dist = CommonDatautils.findDistrict(null, d['B'], d['C'], "district")
//// d['E'] = dist.city
//// }
////
// if(d['F']){
// def dist = CommonDatautils.findDistrictById(d['F'])
// if(dist){
// d['G'] = dist.status
// d['H'] = dist.name_cat
// d['I'] = dist.estate_type2
// }
// }
//
//// def dist = CommonDatautils.findDistrictById(d['A'])
//// if(dist){
//// d['B'] = dist.region
//// d['C'] = dist.name
//// d['J'] = dist.status
//// }
//// def newRegion = RegionData.formatRegion(d['A'], d['B'])
//// d['F'] = newRegion
////// def newRegion = RegionData.formatRegion(d['A'], d['B'])
////// d['F'] = newRegion
//},0)
//ExcelUtils.modify(fileName, {d->
@@ -1373,30 +1379,30 @@ def findPgDealPrice(c, pgsqlInstance, startDate){
return count
}
citys = CcbCommonUtils.citys_all
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
def rows = []
def headers = []
headers << "城市"
headers.addAll(dates)
rows << headers
for(def c : citys){
def row = []
row << c
def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
// for(def d : dates){
def startDate = "2022-07-01"
def count = findPgDealPrice(c, pgsqlInstance, startDate)
row << count*100/disCount
// def map = findSampleRate(pgsqlInstance, c, d)
// row << map.third
// }
rows << row
}
ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
//citys = CcbCommonUtils.citys_all
//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"]
//def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306]
//def rows = []
//def headers = []
//headers << "城市"
//headers.addAll(dates)
//rows << headers
//for(def c : citys){
// def row = []
// row << c
// def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c))
//// for(def d : dates){
// def startDate = "2022-07-01"
// def count = findPgDealPrice(c, pgsqlInstance, startDate)
// row << count*100/disCount
//
//// def map = findSampleRate(pgsqlInstance, c, d)
//// row << map.third
//// }
// rows << row
//}
//ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方")
@@ -1414,6 +1420,31 @@ def findSampleRate(pgsqlInstance, cityName, yearmonth){
//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\房源\\仙桃.xlsx", {d->
// def dist = CommonDatautils.findDistrict(d['A'],d['B'],d['C'], "district")
// def flag = null;
// if(!dist){
// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'],d['B'],d['C'],"district")
// flag = "映射别名匹配"
// if(!dist) {
// def rets = CommonDatautils.findDistrictByAlias(d['A'],d['B'],d['C'], "district")
// if(rets && rets.size()==1){
// flag = "别名匹配"
// dist = rets[0]
// }
// }
// }
// if(dist){
// d['D'] = dist.status
// d['E'] = dist.estate_type2
// d['F'] = flag
// d['G'] = dist.name
// }
//
//})
@@ -54,7 +54,7 @@ class ThirdPartyPriceBatch extends BaseBatch {
// ps.addBatch(id)
// }
// }
//
def citys = CcbCommonUtils.getRunningCitys(args)
def rows = []
def index = 0
@@ -93,13 +93,13 @@ class TradeStatLog {
def run(){
def citys = CcbCommonUtils.citys_all;
def rets = []
rets << ["城市","23年4月总量","23年5月总量","23年6月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
rets << ["城市","23年10月总量","23年11月总量","23年12月总量","占比","贝壳","链家", "我爱我家","幸福里APP","中原"]
def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"]
//def startDate = "2021-05-01"
def startDate = "2023-04-01"
def date = "2023-05-01"
def endDate = "2023-06-01"
def newDate = "2023-07-01"
def startDate = "2023-10-01"
def date = "2023-11-01"
def endDate = "2023-12-01"
def newDate = "2024-01-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
@@ -499,8 +499,8 @@ class TradeStatLog {
def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"]
def rets = []
rets << headers
def startDate = "2022-03-01"
def endDate = "2022-04-01"
def startDate = "2023-08-01"
def endDate = "2023-10-01"
for(def city : citys){
def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
@@ -37,44 +37,47 @@ class ExportAllPgData {
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
////
//// //第三方网站权重表
//////
////// //第三方网站权重表
// exportWebsiteRate();
//
////
// exportGuidedPrice()//随人工改变//
// exportCommunityGuidedPrice()//随政策//
// //处理小区情况
//// //处理小区情况
// exportCommunityStaging()//没cityid
// exportBigdataCommunity()//
//
////
// exportCommunityBuildType()//
//
exportThirdPartyPrice2(new_dists)//
exportBigdataForsale(new_dists) //现在按updatetime导出
////
// exportThirdPartyPrice2(new_dists)//
// exportBigdataForsale(new_dists) //现在按updatetime导出
// exportBigdataDeal(new_dists) //2022-07-01
//计算结果相关
//
//
// //计算结果相关
// exportAvmEvaluationGJNB()
// exportAVMEvaluationItemGJNB()
//
////
// exportCityPrice()
// exportRegionPrice()
exportCommunityBasePrice()
exportCommunityPdescribeGj()
exportCommCaseDetailGj()
// exportCommunityBasePrice()
// exportCommunityBasePrice1(curMonth)
// exportCommunityPdescribeGj()
// exportCommCaseDetailGj()
//
// exportInOk()
exportInOk()
////
// SSHHelperTest jschCommand = new SSHHelperTest();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
SSHHelperTest jschCommand = new SSHHelperTest();
System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}"));
}
def exportEvaluationRateData() {
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h";
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from base.t0831_evaluate_rate_gj_h";
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
@@ -102,7 +105,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'";
def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
@@ -133,7 +136,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'";
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from base.t0831_bigdata_district_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_admndiv_id
@@ -164,7 +167,7 @@ class ExportAllPgData {
}
def exportWebsiteRate(){
def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h";
def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_website_config_gj_h";
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
@@ -194,7 +197,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'";
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
@@ -227,7 +230,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}'";
def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from base.t0831_community_guide_price_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
@@ -307,7 +310,7 @@ class ExportAllPgData {
def exportCommunityStaging() {
def allRows = []
def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h";
def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_community_staging_gj_h";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_instm_id
@@ -330,7 +333,7 @@ class ExportAllPgData {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'";
def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from base.t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
@@ -363,12 +366,12 @@ class ExportAllPgData {
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
@@ -424,17 +427,17 @@ class ExportAllPgData {
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'";
allRows.addAll(dealData(sql))
}
}
@@ -535,7 +538,7 @@ class ExportAllPgData {
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}";
"from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
@@ -560,7 +563,7 @@ class ExportAllPgData {
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
"from base.t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
def rets2 = getHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true);
@@ -17,7 +17,7 @@ class ExportPgsqlBaseEvaluation {
def database = "base"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\"
static main(args) {
@@ -41,11 +41,10 @@ class ExportPgsqlEvaluationResult {
exportRegionPrice(curMonth)
exportCommunityBasePrice(curMonth)
// exportCommunityBasePrice1(curMonth)
exportCommunityPdescribeGj(curMonth)
exportCommCaseDetailGj(curMonth)
//
exportInOk()
//
SSHHelperTest jschCommand = new SSHHelperTest();
@@ -61,8 +61,8 @@ class BigdataCitypriceBatch extends BaseBatch{
def dealStartDate = DateUtils.addMonth(DateUtils.parseFromTightByPattern(startDate.toString(), "yyyyMMdd"), -1)
def lastYearDate = DateUtils.addMonth(startDate.toString(),-12, "yyyyMMdd").toInteger();
// cityRun(citys, startDate, endDate, cityCodeMap)
// regionRun(citys, startDate, endDate, cityCodeMap)
cityRun(citys, startDate, endDate, cityCodeMap)
regionRun(citys, startDate, endDate, cityCodeMap)
districtRun(citys, startDate, endDate, dealStartDate, lastYearDate, cityCodeMap)
}
@@ -52,6 +52,8 @@ class ExportDistrictRongHe extends BaseBatch{
exportBigdataBuildingFJW()//
exportBigdataCommfeeGj()//
exportGuidedPrice()//10月10号再同步数据
exportBigdataCommbuildstruGJ()//
exportBigdataCommhousetypeGJ()//
exportBigdataCommbuildyearGj()//
@@ -188,6 +190,46 @@ class ExportDistrictRongHe extends BaseBatch{
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMUNITY_FJW ÉÏ´«FtpÍê³É......"
}
def exportGuidedPrice() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from base.t0831_guided_price_gj_h where urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_bds_id
data << r.comm_nm
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_drc_dt
data << r.gujia_bds_drc_prc
data << r.gujia_add_psn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "GUIDED_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "GUIDED_PRICE_GJ 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def allRows = []
for(def city : citys) {
@@ -19,10 +19,12 @@ class PocTest {
static main(args) {
def task = new PocTest();
task.run();
// task.run();
// task.export()
// task.distance()
}
def export(){
ExcelUtils.modify("D:\\Êý¾Ý\\ÆäËü\\¹ãÖÝ.xlsx", { row ->
def dist = CommonDatautils.findDistrictById(row["B"])
@@ -156,7 +158,6 @@ class PocTest {
if(commid){
def dist = CommonDatautils.findDistrictById(commid.trim())
if(dist){
// row['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
row['R'] = priceObj.avg_price
@@ -1,8 +1,12 @@
package com.fangjia.script.commData
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import groovy.json.JsonSlurper
@@ -32,6 +36,7 @@ import org.elasticsearch.index.query.QueryBuilders
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
class CommonDatautils {
@@ -83,8 +88,17 @@ class CommonDatautils {
public static final DBCollection REGION_INFO_CONN = MongoUtils.getCollectionByName("fangjia_data_ana", "region_info", MongoUtils.MongoDB_Domain_mongo);
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
public static final DBCollection confirm_price_date = MongoUtils.getCollectionByName("fangjia", "confirm_price_data", "mongo.db.fangjia.com")
public static final DBCollection TOWN_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "town_polyline")
public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009";
public static final DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL)
public static final DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL)
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
static def DIRECTIONS = ["¶«±±", "Î÷±±", "¶«ÄÏ", "Î÷ÄÏ", "Äϱ±", "¶«Î÷", "¶«", "ÄÏ", "Î÷", "±±"]
private static final HashMap<String, List<String>> FITMENTS_TYPES_MAPS = new HashMap<String, List<String>>()
static{
@@ -227,12 +241,11 @@ class CommonDatautils {
public static def findRegionDistrictCount(def cat, def city, def region){
DBObject q = new BasicDBObject();
q.put("visible", 0);
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
q.put("region", region);
DSCursor cursor = getDataStoreConnection().find(q);
return cursor.count();
return DbUtils.seaweed.count(q);
}
public static def findAllFiledDistrict(def cat, def city){
@@ -246,12 +259,13 @@ class CommonDatautils {
List<DBObject> dbs = cursor.getResults();
return dbs;
}
public static def findAllDistricts(def city, def cat, def fields){
public static def findAllDistrictsFromMg(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible", 0);
q.put("status", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","泰瑞家园")
DBObject keys = new BasicDBObject();
if(fields){
@@ -260,7 +274,7 @@ class CommonDatautils {
}
}
DSCursor cursor = getDataStoreConnection().find(q, keys).batchSize(2000)
def cursor = DbUtils.seaweed.find(q, keys).batchSize(2000)
List<DBObject> rets = []
for(def obj : cursor){
rets << obj
@@ -298,7 +312,11 @@ class CommonDatautils {
q.put("cat", cat);
q.put("city", city);
if(startDate){
q.put("m_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def datequery = new BasicDBObject('$gte', startDate)
if(endDate){
datequery.append('$lt',endDate)
}
q.put("m_date", datequery)
}
DBObject keys = new BasicDBObject();
@@ -784,10 +802,33 @@ class CommonDatautils {
}
public static def findAllDistricts(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city)
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllDistricMaptByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
@@ -797,7 +838,7 @@ class CommonDatautils {
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
def retMap = [:]
for(def d : dbsour){
retMap.put(d._id.toString(), d)
@@ -806,27 +847,27 @@ class CommonDatautils {
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name","Íò´ï»ª¸®Ò»ÆÚ")
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
// public static def findAllDistrictByCity(def city, def cat, def fields){
//
// DBObject q = new BasicDBObject();
// q.put("status",0)
// q.put("cat", cat);
// q.put("city", city);
//// q.put("name","万达华府一期")
//
// DBObject keys = new BasicDBObject();
// if(fields){
// for(def field : fields){
// keys.put(field, 1);
// }
// }
// def dbsour = SEAWEED_COLL.find(q, keys).batchSize(2000)
// def rets = []
// for(def d : dbsour){
// rets << d
// }
// return rets;
// }
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
@@ -1731,14 +1772,14 @@ class CommonDatautils {
}
public static def queryDistrictPrice(city, region, name, weekend){
public static def queryFirstDistrictPrice(city, region, name){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
queryCondition.put("weekend", weekend);
// queryCondition.put("weekend", weekend);
DBCursor cur = DIST_STAT_COLL.find(queryCondition)
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", -1)).limit(1)
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def dbo = null;
while(cur.hasNext()) {
@@ -2206,6 +2247,21 @@ class CommonDatautils {
}
public static def queryAllDistrictPrice(city, startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put('weekend', startDate);
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1))
cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rMap = [:];
while(cur.hasNext()) {
DBObject dbo = cur.next()
rMap.put(city + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo)
}
return rMap;
}
public static def queryAllDistrictPrice(startDate){
DBObject queryCondition = new BasicDBObject()
queryCondition.put('weekend', startDate);
@@ -2604,6 +2660,7 @@ class CommonDatautils {
q.put("cat", cat);
q.put('lat2', new BasicDBObject(QueryOperators.LTE, maxLat).append(QueryOperators.GTE, minLat));
q.put('lng2', new BasicDBObject(QueryOperators.LTE, maxLng).append(QueryOperators.GTE, minLng));
q.put("visible", 0)
def cur = getDataStoreConnection().find(q);
def ret = [];
for(def obj : cur){
@@ -3862,42 +3919,6 @@ class CommonDatautils {
return avg_price
}
static def findThirdPartyPrice(def city, def region, def name, def startDate, def endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("lianjia_esf_price", db.beike_esf_price)
}
}
return map
}
/**
@@ -3941,5 +3962,381 @@ class CommonDatautils {
return files;
}
static def findLogs(userId, requestUri, startDate, endDate, code){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
if(startDate){
def startTime = DateUtils.parseFromTightYYYYMMDD(startDate)
def endTime = DateUtils.parseFromTightYYYYMMDD(endDate)
def date_range_cdate = QueryBuilders.rangeQuery("c_date").from(startTime.getTime()).to(endTime.getTime()).includeLower(true).includeUpper(false);
queryBuilder.must(date_range_cdate);
}
if(code){
queryBuilder.must(QueryBuilders.termQuery("code", code))
}
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
return rets.size()
}
static def findLogsBySql(userId, requestUri, startDate, endDate, code){
String sql = "select count(*) as total from open_uri_log where user_id=${userId} and request_uri='${requestUri}' "
if(startDate){
sql = sql + "and c_date >= '${startDate}' and c_date < '${endDate}'"
}
if(code){
sql = sql + " and code=${code}"
}
def total = null
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
return total;
}
static def findNewAmapDetailBySql(city){
String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
def point = data.location
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
rets << ["data_id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location]
})
return rets;
}
static def findAmapDetailBySql(city){
String sql = "select * from amap_poi where cityname='${city}' and status=0"
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
// if(data.name.contains('华山小区G栋')){
def point = data.location
// def tag = null
// if(data.get("business.keytag")){
// tag = data.get("business.keytag")
// }
def business_area = null
if(data.get("business.business_area")){
business_area = data.get("business.business_area")
}
def location = [:]
location.lon = DataHandleUtils.getDoubleRound(6, point.x)
location.lat = DataHandleUtils.getDoubleRound(6, point.y)
rets << ["id":data.id,"province":data.pname,"city":data.cityname, "district":data.adname, "name":data.name, "address":data.address, "cat":data.type, "location":location, "business_area":business_area]
// }
})
return rets;
}
//用sql有些字段取不出来
static def findDetailLogsBySql(userId, requestUri){
String sql = "select * from open_uri_log where user_id=${userId} and request_uri='${requestUri}' and code!=200"
sql = sql + " order by c_date desc"
def total = null
def rets = []
ElasticUtils.executeSqlQuery(sql,{data->
def ret = []
println data.toString()
ret << data.get("parameter_map.city")
// ret << data.get("parameter_map.district")
// ret << data.get("parameter_map.keyword")
// ret << data.get("parameter_map.name")
ret << data.get("parameter_map.address")
// ret << data.get("parameter_map.size")
ret << data.get("msg")
ret << data.get("code")
ret << data.get("c_date").toString()[0..10]
rets << ret
})
return rets;
}
static def findNewAmapDetail(city) {
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("fjw_city", city))
def rets = ElasticUtils.executeSimpleQuery("192.168.0.12", "amap_poi", queryBuilder);
def results = []
if(rets) {
for (int k = 0; k < rets.size(); k++) {
def data = rets[k];
println data
}
}
return results;
}
static def findLogs2(userId, requestUri){
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", userId)).must(QueryBuilders.termQuery("request_uri", requestUri))
queryBuilder.mustNot(QueryBuilders.termQuery("code", 200))
def rets = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
def results = []
if(rets) {
for (int k = 0; k < rets.size(); k++) {
def data = rets[k];
println data
def ret = []
println data.toString()
ret << data.parameter_map.city
ret << data.parameter_map.district
ret << data.parameter_map.name
ret << data.parameter_map.address
ret << data.msg
ret << data.code
ret << data.c_date.toString()[0..10]
results << ret
}
}
return results
}
static def httpSaveData(def newUrl,def params){
def header = [:]
header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=")
doPost(newUrl,header, params, true, "utf-8")
}
static def doPost(def targetUrl, def headers, def params, def isJson, def charset){
StringBuilder buffer = new StringBuilder();
try{
URL url = new URL(targetUrl);
HttpURLConnection conn = (HttpURLConnection) url.openConnection();
conn.setRequestMethod("POST");
conn.setDoOutput(true);
conn.setUseCaches(false);
conn.setConnectTimeout(5000);
conn.setReadTimeout(5000);
conn.setRequestProperty("User-Agent", UserAgents.get());
String param = null;
BufferedWriter mWriter = null;
InputStreamReader inStream = null;
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
conn.setRequestProperty(head, headers.get(head));
}
} else {
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" );
}
if (isJson) {
conn.setRequestProperty("Content-Type", "application/json; charset=" + charset);
param = JacksonUtil.beanToJson(params)
}
conn.connect();
//println param.toString()
//输出流
if (StringUtils.isNotEmpty(param)) {
mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset));
mWriter.write(param);
mWriter.flush();
mWriter.close();
}
if (conn.getHeaderField("Content-Encoding") != null
&& conn.getHeaderField("Content-Encoding").equals("gzip")) {
inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset);
} else {
inStream = new InputStreamReader(conn.getInputStream(), charset);
}
char[] bf = new char[1024];
int len;
while ((len = inStream.read(bf)) != -1) {
buffer.append(bf, 0, len);
}
} catch (Exception e) {
e.printStackTrace()
}
return buffer.toString();
}
/**
* 获取小区周边一公里小区数量
* @param city
* @param lng
* @param lat
* @return
*/
static getDistrictArround(city, lng, lat) {
try {
def query = new BasicDBObject()
query.append("cat", "district")
query.append("city", city)
query.append("visible", 0)
DBObject sdq = new BasicDBObject();
sdq.put("sd.point", lat + "," + lng);
sdq.put("sd.sfield", "lat_lng");
sdq.put("sd.distance", String.valueOf((double) 1000 / 1000.0));
query.append('$sd', sdq)
return DbUtils.seaweed.find(query).toList()
} catch (Exception e) {
e.printStackTrace()
}
return 0
}
static def findThirdPartyPrice(def city, def region, def name, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", new BasicDBObject('$gte', startDate).append('$lte',endDate))
def dbcursor = thirdpartColl.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
def key = db.city+"##"+db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
map.put(key, dataMap)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
}
return map
}
static def findGXDPrice(def city, region, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("source", "GXD")
def dbcursor = confirm_price_date.find(q).sort(new BasicDBObject("c_date", -1)).limit(1)
def price = null
for(def d : dbcursor){
price = d.price
}
return price
}
static def findTownRange(city){
def q = new BasicDBObject()
q.put("city",city)
def dbcursor = TOWN_POLYLINECOLL.find(q)
def townMap = [:]
for(def d : dbcursor){
def pointList = InsidePolygon.createPoint3(d.polyline);
townMap.put(d.fjw_region +"##"+ d.town, pointList)
}
return townMap
}
static def queryFullNameCity(){
def q = new BasicDBObject()
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
def fjwcity = null
if(d.level == "城市"){
fjwcity = RegionData.formatCity(d.city)
if(!fjwcity){
if(d.city == "市辖区"){
fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
d.city = d.province
}
}
}
}else if(d.level == "区域"){
if(d.administer == "省直辖县级行政区划"){
fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
d.city = d.district
}
}
}
if(fjwcity){
retMap.put(fjwcity, d)
}else{
println d.toString() +" 格式化不成功!"
}
}
return retMap;
}
static def queryDistNameCity(){
def q = new BasicDBObject()
q.put("level","区域")
def dbcursor = administrative_division_100_CONN.find(q)
def retMap = [:]
for(def d : dbcursor){
if(d.city == "省直辖县级行政区划"){
continue
}else if(d.city == "市辖区"){
def fjwcity = RegionData.formatCity(d.province)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}else{
def fjwcity = RegionData.formatCity(d.city)
if(fjwcity){
def fjwdist = RegionData.formatRegion(fjwcity, d.district)
retMap.put(fjwcity+"#"+fjwdist, d)
}
}
}
return retMap;
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", cat);
q.put("city", city);
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = SEAWEED_MONGO_COLL.find(q,keys).batchSize(2000)
def rets = []
def ids = []
for(def d : dbsour){
rets << d
}
println "[${city}] 小区量:"+rets.size()
return rets;
}
}
@@ -24,16 +24,16 @@ class LoudongDataCoverBatch extends BaseBatch {
static Jms jms = Jms.create('tcp://jms.db.fangjia.com:61616');
public DBCollection mongoColl = MongoUtils.getCollectionByName("fangjia", "house_loudong", "mongo.db.fangjia.com")
public static final def EMAILS = [
"duxiaoyu@fangjia.com",
// "duxiaoyu@fangjia.com",
// "ruanyadong@fangjia.com",
// "xiamin@fangjia.com",
// "zhutianli@fangjia.com",
"joe@fangjia.com",
"yukai@fangjia.com",
"jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com",
"shijing@fangjia.com",
"xiaohai@fangjia.com",
// "joe@fangjia.com",
// "yukai@fangjia.com",
// "jiangxiaochun@fangjia.com",
"gaojuan@fangjia.com"
// "shijing@fangjia.com",
// "xiaohai@fangjia.com",
]
static def FIELDS_MAP = [
@@ -27,8 +27,8 @@ class ApiDailyCount extends BaseBatch{
private static final Jms jms = Jms.create('tcp://monitor.jms.db.fangjia.com:61616')
def static emails = [
'xuqi@fangjia.com',
'zongzhimin@fangjia.com'
'wangwenyan@fangjia.com',
'gaojuan@fangjia.com'
]
static void main(String[] args){
@@ -0,0 +1,97 @@
package com.fangjia.script.district
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
@BatchOwner(Watcher.GaoJuan)
class DistrictFacilities2Batch extends BaseBatch {
public DBCollection onlineColl = MongoUtils.getCollectionByName("fangjia", "district_arround_poi", "mongo.db.fangjia.com")
public static void main(String[] args) {
//小区周边数据更新
this.runBatch(new DistrictFacilities2Batch(), args);
}
@Override
public void process(Object args) {
def cities = CommonDatautils.findAllCity()
for (def city : cities) {
def ret = CommonDatautils.getAllDistricts("district", city)
def index = 0;
for (def dbo : ret) {
index++;
println index + "/" + ret.size() + " " + dbo.city + " " + dbo.name
String region = dbo.get('region')
String districtName = dbo.get('name')
if (!dbo.lat2 && !dbo.lng2) {
continue;
}
def poms = findPoms(dbo.lat2 / 10000000000.0, dbo.lng2 / 10000000000.0);
DBObject object = new BasicDBObject();
object.put("hospital", unique(processType(poms.get('医院'), 1000, null, ['宠物', '动物', '口腔', '牙科', '美容', '齿科', '眼科', '号楼', '住院部', 'Pure', 'Tokushinkai', '假牙制作', '镶牙', '咪', '喵'])))
object.put("shop", processType(poms.get('超市'), 1000, ['便民商店', '便利店'], ['杂货店', '烟酒', '爱鑫', '超市便利店']))
object.put("university", processType(poms.get('学校'), 1000, ['大学'], ['食堂', '办公室', '德威', '早教', '夕阳红', '公司']))
object.put("middle_school", processType(poms.get('学校'), 1000, ['中学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
object.put("primary_school", processType(poms.get('学校'), 1000, ['小学'], ['食堂', '办公室', '早教', '夕阳红', '公司']))
object.put("kindergarten", processType(poms.get('学校'), 1000, ['幼儿园', '托儿所'], ['食堂', '亲子', '办公室', '夕阳红', '公司']))
object.put("food_shop", processType(poms.get('餐饮'), 500, ['餐饮服务']))
def market = [];
if (poms.get('商场')) {
market = market + poms.get('商场');
}
if (poms.get('超市')) {
market = market + poms.get('超市');
}
object.put("supermarket", processType(market, 1000, ['购物中心', "超市", "好又多"], ['公司', '村村寨寨', '伍缘折扣', '杂货店']))
object.put("bank", processType(poms.get('银行'), 1000, ['银行'], ['自动提款机']))
object.put("atm", processType(poms.get('ATM'), 1000, ['自动提款机', "atm", "ATM"]))
updateToDistrictFacilitys(city, region, districtName, object)
updateToDistrictFacilitysScore(city, region, districtName)
}
}
}
def findPoms(def lat, def lng) {
def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000)
DBObject q = new BasicDBObject()
q.put('lat', new BasicDBObject(QueryOperators.LTE, map.maxLat).append(QueryOperators.GTE, map.minLat));
q.put('lng', new BasicDBObject(QueryOperators.LTE, map.maxLng).append(QueryOperators.GTE, map.minLng));
def cursor = POI_COLL.find(q);
def resMap = [:];
for (def dbo : cursor) {
def keyword = dbo.keyword;
def details = resMap.get(keyword);
if (!details) {
details = [];
resMap.put(keyword, details);
}
def dMap = [:];
dMap.name = dbo.name;
dMap.lat = dbo.lat;
dMap.lng = dbo.lng;
dMap.distance = com.fangjia.script.util.GeoUtils.calculateDistance(lat, lng, dbo.lat, dbo.lng);
details << dMap;
}
resMap.each { k, v ->
println "[${k}] " + v;
}
return resMap;
}
}
@@ -75,7 +75,7 @@ class DistrictHouseBatch extends BaseBatch {
sellQuery.put('$facet', facet)
sellQuery.put('city', city)
sellQuery.put('cat',cat)
sellQuery.put('visible',0)
// sellQuery.put('visible',0)
DSCursor sellDSCursor = SEABIRD.find(sellQuery).skip(0).limit(0)
List<FacetResult> sellFacetResult = sellDSCursor.getFacetResults();
if(sellFacetResult){
@@ -82,7 +82,7 @@ class LngLatEachWeek extends BaseBatch {
href = "<a href='http://static.fangjia.com/stc/fs/${href}'>点击下载附件</a>"
jms.sendMail("caoyuyan@fangjia.com","未匹配上的小区经纬度",href)
jms.sendMail("wangchencen@fangjia.com","未匹配上的小区经纬度",href)
jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href)
// jms.sendMail("sunjian@fangjia.com","未匹配上的小区经纬度",href)
}
public static Map<String, Object> getLngLatByAddress(String city, String address){
@@ -1,5 +1,6 @@
package com.fangjia.script.fuzhou
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
@@ -38,9 +39,8 @@ class AmapAddressUpdate {
static main(args) {
def task = new AmapAddressUpdate()
def path = "d:\\数据\\小区\\福州小区地址与高德地图V2.xlsx"
def addr = task.formatAddress("福州", "北三环路201号")
println addr
def path = "d:\\数据\\小区\\重庆小区地址与高德地图V1.xlsx"
// task.run(path)
// task.similarMatch(path)
// task.addressRun(path)
@@ -50,11 +50,15 @@ class AmapAddressUpdate {
// task.addrAndLocationDistance(path)//地址经纬度与库中经纬度比较
// task.updateData(path)
// task.updateLocation(path)//更新经纬度
task.matchNoSameRegionDist()
}
def run(path){
def citys = ["福州"]//CommonDatautils.findAllCity()
//"三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田"
def citys = ["重庆"]//CommonDatautils.findAllCity()
def allrets = []
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度"]
allrets << ["城市","区域","名称","地址","地址别名","高德区域","高德名称","高德地址","高德地址格式化","高德经度","高德纬度","得分","匹配类型","小区经纬度与高德距离","操作","来源","地址与小区经纬度距离","备注","经纬度是否修改","高德类型"]
for (def city : citys) {
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address", "addr_alias","lat", "lng"])
def rets = findAmapDetail(disticts)
@@ -63,6 +67,17 @@ class AmapAddressUpdate {
ExcelUtils.write(path, allrets, "数据")
}
def matchNoSameRegionDist(){
def citys = ["三明","宁德","泉州","厦门","福州","漳州","南平","龙岩","莆田","重庆"]
for (def city : citys) {
def disticts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name","address","estate_type2", "addr_alias","lat", "lng"])
def rets = findDiffRegionAmapDetail(disticts)
ExcelUtils.write("D:\\数据\\小区\\区域有问题小区.xlsx", rets, city)
}
}
def amapRun(){
def city = "福州市"
def datas = findAllAmapByCity(city)
@@ -177,7 +192,15 @@ class AmapAddressUpdate {
row << formatAddress(d.city, dbo.address)
row << dbo.lng
row << dbo.lat
row << 1
row << dbo.match_type
if(d.lng && dbo.lng){
def distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
row << distance
}else{
row << ""
}
}else{
row << ""
row << ""
@@ -186,6 +209,8 @@ class AmapAddressUpdate {
row << ""
row << ""
row << ""
row << ""
row << ""
}
println totalCount.get() +' '+row
rets << row
@@ -201,6 +226,75 @@ class AmapAddressUpdate {
return rets
}
def findDiffRegionAmapDetail(districts){
def rets = Collections.synchronizedList(new ArrayList());
rets << ["城市","区域","名称","地址","物业类型","地图"]
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def dbos = getAmapLocations(d.city+"市", d.name)
if(!dbos){
def newName = d.name
if(d.name.toString().endsWith("别墅") || d.name.endsWith("商住楼") || d.name.endsWith("钢混") || d.name.endsWith("混合")){
newName = newName.replace("(非钢混)","").replace("钢混","").replace("(混合)","").replace("混合","").replace("别墅","").replace("商住楼","")
}
dbos = getAmapLocations(d.city+"市", newName)
}
if(dbos){
def sameFlag = false
for(def dbo : dbos){
def newRegion = RegionData.formatRegion(d.city, dbo.region)
if(d.region == newRegion){
sameFlag = true
continue
}
}
if(!sameFlag){
def row = []
row << d.city
row << d.region
row << d.name
row << d.address
row << d.estate_type2
def str = ""
for(def dbo : dbos){
def distance = null
if(d.lng && dbo.lng){
distance = com.fangjia.core.util.GeoUtils.calculateDistance(d.lng/10000000000d, d.lat/10000000000d, dbo.lng.toString().toDouble(), dbo.lat.toString().toDouble());
}
str = str + dbo.region+"-"+dbo.name+"-"+dbo.address+"-"+dbo.type+"-"+distance+"\n"
}
row << str
rets << row
}
}
println totalCount.get()+" "+d.city
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets
}
def formatAddress(def city, def address){
def jsonObj = FangjiaClientUtils.addressStructure(city, address)
def address_str = null
@@ -267,6 +361,38 @@ class AmapAddressUpdate {
}
def getAmapLocations(city, name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi_detail WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and type like '%住宅%'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def list = []
while (rs.next()) {
// println rs.getString("adname")+" "+rs.getString("name") +" "+rs.getString("address") +" "+rs.getString("type")
def mapName = rs.getString("name").replace("·","")
if(mapName == name){
def map = [:]
map.name = rs.getString("name")
map.address = rs.getString("address")
map.formatAddress = AddressUtil.formatAddress(city, map.address)
map.region = rs.getString("adname")
map.type = rs.getString("type")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
list << map
}
}
return list;
}
@@ -451,9 +577,11 @@ class AmapAddressUpdate {
}
def updateLocation(path){
ExcelUtils.multiThreadingModify(path, { row ->
def index = 0
ExcelUtils.modify(path, { row ->
if(row['R'] && row['R']=="是"){
index++
def dist1 = CommonDatautils.findDistrict(row['A'], row['B'], row['C'], "district")
if(dist1){
def udata = new BasicDBObject()
@@ -514,8 +642,9 @@ class AmapAddressUpdate {
row['J'] = dbo.lng
row['K'] = dbo.lat
row['L'] = dbo.score
row['M'] = dbo.distance
row['S'] = dbo.type
row['N'] = dbo.distance
row['M'] = dbo.match_type
row['T'] = dbo.type
}
}
}
@@ -539,7 +668,7 @@ class AmapAddressUpdate {
def mapName = rs.getString("name").replace("·","")
println name +" "+ mapName
if(rs.getString("adname").contains(region)){
if(rs.getString("adname") && rs.getString("adname").contains(region)){
map.name = rs.getString("name")
map.address = rs.getString("address")
@@ -97,7 +97,7 @@ class DistrictInfoExportBatch extends BaseBatch{
saveOrUpdateFzDist(districts)
if(day == 15){
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), endMDate);
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), null);
remarkRemoveDist(removeDists)
}else{
def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, startMDate, endMDate);
@@ -201,6 +201,9 @@ class DistrictInfoExportBatch extends BaseBatch{
def udate = new BasicDBObject()
udate.put("status",1)
udate.put("m_date", rd.m_date)
udate.put("region", rd.region)
udate.put("name", rd.name)
udate.put("estate_type2", rd.estate_type2)
FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate));
}
}
@@ -47,19 +47,29 @@ class DistrictPriceExportBatch extends BaseBatch{
def ret = createNewDatas("福州")
saveToDB(ret)
def results = selectLastResultDatas(ret);
def fileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".csv"
def dataCodeFileName = "/fzbank/output/FJW_DISTRICT_STAT_"+curDate+".sha512"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".csv"
dataCodeFileName = "D:\\数据\\福州\\FJW_DISTRICT_STAT_"+curDate+".sha512"
}
def sb = appendResult(headers, ret)
def sb = appendResult(headers, results)
DataHandleUtils.outPutFile(fileName, sb, charset);
FzFtpUtils.ftpUpload(fileName)
//生成加密数据文件并上传
FzFtpUtils.uploadSha512Data2(fileName, dataCodeFileName, charset);
}
def selectLastResultDatas(def rets){
def rows = []
for(def r : rets){
rows << [r[0],r[1], r[2], r[3], r[4],r[5], r[6]]
}
return rows;
}
def appendResult(def headers, def ret){
@@ -121,6 +131,7 @@ class DistrictPriceExportBatch extends BaseBatch{
for(def rMap : districts){
def curPrice = 0
def dbo = null
if(curPriceMap.containsKey(rMap.get("_id").toString())){
dbo = curPriceMap.get(rMap.get("_id").toString())
curPrice = dbo.avg_price
@@ -147,6 +158,29 @@ class DistrictPriceExportBatch extends BaseBatch{
m << 0
}
m << curDate.toInteger();
if(curPrice && dbo){
if(dbo.value_type == 1){
m << "指导价"
}else if(dbo.value_type == 2){
m << "自身样本"
}else if(dbo.value_type == 3){
m << "第三方"
}else if(dbo.value_type == 4){
m << "交易数据"
}else if(dbo.value_type == 5){
m << "周边楼盘"
}else if(dbo.value_type == 6){
m << "推导"
}else if(dbo.value_type == 9){
m << "神经网络"
}else if(dbo.value_type == 10){
m << "无方式"
}else{
m << ""
}
}else{
m << "无"
}
result << m
}
return result
@@ -300,6 +334,7 @@ class DistrictPriceExportBatch extends BaseBatch{
//按comm_id修改,小区名重新赋值,防止区域和小区名修改
udata.put("region", r[3])
udata.put("name", r[4])
udata.put("value_type", r[7])
FZ_DISTRICT_STAT_COLL.update(q, new BasicDBObject('$set', udata))
}else{
def o = new BasicDBObject()
@@ -312,6 +347,7 @@ class DistrictPriceExportBatch extends BaseBatch{
o.put("avg_price", r[5].toInteger())
o.put("month", r[6].toInteger())
o.put("c_date", new Date())
o.put("value_type", r[7])
FZ_DISTRICT_STAT_COLL.save(o)
}
}
@@ -32,6 +32,8 @@ class DistrictSellStatBatch extends BaseBatch {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static final DBCollection SEAWEED_MONGO_COLL = MongoUtils.getCollectionByName("fangjia", "seaweed", "mongo.db.fangjia.com")
public static void main(String[] args) {
runBatch(new DistrictSellStatBatch(), args);
@@ -88,18 +90,21 @@ class DistrictSellStatBatch extends BaseBatch {
}
}else{
def q = new BasicDBObject("city", city).append("cat", "district").append("status", 0)
// q.put("name", "翠园楼")
if(day == 2){
districts = DbUtils.seaweed.find(q, keys).batchSize(2000)
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
}else{
q.put("m_date", new BasicDBObject('$gte',startMDate).append('$lt', endMDate))
districts = DbUtils.seaweed.find(q, keys).batchSize(2000)
districts = SEAWEED_MONGO_COLL.find(q, keys).batchSize(2000)
}
}
println index+" "+city +" "+districts.size()+" "+ (new Date()).toString()
def startDate = DateUtils.getFirstDayOfMonth(new Date(), -1);
def endDate = DateUtils.getLastDayOfMonth(new Date(), -1);
if(districts.size()>0){
def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//ÉϸöÔÂÐ¡Çø¾ù¼Û
def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价
def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价
def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价
def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价
@@ -135,11 +140,12 @@ class DistrictSellStatBatch extends BaseBatch {
def rets = new CopyOnWriteArrayList()
districts.each{dist->
// fixedThreadPool.execute(new Runnable() {
// void run() {
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def key = dist.region +"##" + dist.name
def weekendPriceObj = weekendPriceMap.get(key)
if(weekendPriceObj){
def curPriceObj = curMonthlyPriceMap.get(key)
@@ -151,24 +157,19 @@ class DistrictSellStatBatch extends BaseBatch {
mom = DataUtil.roundDouble(((weekendPriceObj.avg_price - lastMonthPriceObj.avg_price)/(double)lastMonthPriceObj.avg_price),4)
}
def curPrice = null;
if(curPriceObj){
curPrice = curPriceObj.avg_price
}else{
curPrice = weekendPriceObj.avg_price
}
def curPrice = weekendPriceObj.avg_price
if(weekendPriceObj.value_type!=1){
if(dist.city == "福州"){
// if(Math.abs(mom)*100>3){
// def randMarkup = nextDouble(0, 3)
// if(mom < 0){
// mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
// }else{
// mom = DataUtil.roundDouble(randMarkup/100, 4)
// }
// curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
// }
if(Math.abs(mom)*100>3){
def randMarkup = nextDouble(0, 3)
if(mom < 0){
mom = DataUtil.roundDouble(randMarkup*(-1)/100, 4)
}else{
mom = DataUtil.roundDouble(randMarkup/100, 4)
}
curPrice = (int)(lastMonthPriceObj.avg_price*mom + lastMonthPriceObj.avg_price)
}
}else{
if(Math.abs(mom)*100>10){
def randMarkup = nextDouble(0, 10)
@@ -197,7 +198,9 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("commid", dist._id.toString())
data.put("mom", mom)
data.put("yoy", yoy)
data.put("avg_price", curPrice)
data.put("avg_price",curPrice)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
DISTRICT_SELL_STAT.update(new BasicDBObject("_id", curPriceObj._id) , new BasicDBObject('$set', data),true,false)
println "[update]"+totalCount.get()+" "+ dist.city+" "+dist.name
}
@@ -209,6 +212,7 @@ class DistrictSellStatBatch extends BaseBatch {
data.put("estate_type", dist.estate_type2)
data.put("date", curMonth)
data.put("avg_price", curPrice)
data.put("value_type", weekendPriceObj.value_type)
data.put("credit", weekendPriceObj.credit)
data.put("m_date", new Date())
data.put("commid", dist._id.toString())
@@ -224,10 +228,10 @@ class DistrictSellStatBatch extends BaseBatch {
} finally {
latch.countDown()
}
// };
// })
};
})
}
// latch.await()
latch.await()
}
@@ -245,11 +249,12 @@ class DistrictSellStatBatch extends BaseBatch {
}
def findPriceMap(city, weekend){
def findPriceMap(city, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DISTRICT_STAT.find(q)
// q.put("district_name","金福花园")
q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate))
def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1))
def priceMap = [:]
for(def d : dbcursor){
if(city == "福州"){
@@ -257,7 +262,9 @@ class DistrictSellStatBatch extends BaseBatch {
d.avg_price = d.avg_price_2b
}
}
priceMap.put(d.region +"##"+d.district_name, d)
if(!priceMap.containsKey(d.region +"##"+d.district_name)){
priceMap.put(d.region +"##"+d.district_name, d)
}
}
return priceMap
}
@@ -22,6 +22,10 @@ class FzDataResultStat {
public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com")
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl)
static main(args) {
def task = new FzDataResultStat()
task.priceMarkupRun();
@@ -51,59 +55,67 @@ class FzDataResultStat {
})
}
def priceMarkupRun(){
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")+"15");
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// List<String> list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv"))
// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"])
def dists = getFzDistricts("福州", ["status"])
def idMaps = [:]
for(def d : dists){
idMaps.put(d._id.toString(), d.status==0?"有效":"无效")
}
def list1 = findFzDistrictStat(Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")))
def distkeys = []
def map1 = [:]
for(def s : list1){
def row = s.split("@\\|@")
def key = row[2]+"##"+row[3]+"##"+row[4]
for(def obj : list1){
def key = obj.city + "##" + obj.region + "##" + obj.name
if(!distkeys.contains(key)){
distkeys << key
}else{
println "重复:"+key
}
map1.put(key, row)
map1.put(key, obj)
}
List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
// List<String> list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv"))
def list2 = findFzDistrictStat(lastMonth)
def map2 = [:]
for(def s : list2) {
def row = s.split("@\\|@")
def key = row[2] + "##" + row[3] + "##" + row[4]
map2.put(key, row)
for(def obj : list2) {
def key = obj.city + "##" + obj.region + "##" + obj.name
map2.put(key, obj)
}
def rows = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"]
map1.each {k,list->
def row = []
rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"]
def index = 0
map1.each {k,obj1->
def keys = k.split("##")
def lastObj = map2.get(k)
def lastPrice = null
if(lastObj){
lastPrice = lastObj[5]
}
def lastPrice = map2.get(k)
def markup = null
try{
if(list && lastPrice && lastPrice.toInteger()>0){
markup = (list[5].toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger()
if(obj1 && lastPrice && lastPrice.avg_price.toInteger()>0){
markup = (obj1.avg_price.toInteger() - lastPrice.avg_price.toInteger())*100.0/lastPrice.avg_price.toInteger()
}
}catch(Exception e){
e.printStackTrace()
}
def dist = CommonDatautils.findDistrictById(list[1])
def flag = "有效"
if(dist && dist.status!=0){
flag = "失效"
def flag = "无效"
if(obj1){
flag = idMaps.get(obj1.comm_id)
}
rows << [keys[0], keys[1], keys[2], list[5].toString(), lastPrice.toString(), markup, flag]
def markup_abs = null
if(markup){
markup_abs = Math.abs(markup)
}
def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs]
rows << row
index++
println row
}
ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据")
@@ -255,4 +267,30 @@ class FzDataResultStat {
}
return files;
}
def findFzDistrictStat(lastMonth){
def q = new BasicDBObject()
q.put("month", lastMonth)
return FZ_DISTRICT_STAT_COLL.find(q).toList()
}
def getFzDistricts(def city, fields){
def q = new BasicDBObject()
q.put("city", city)
q.put("cat", "district")
def rets = []
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
FZ_DIST_COLL.find(q, keys).each{it->
rets << it
}
return rets
}
}
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202306.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202309.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -76,7 +76,7 @@ class ThirdPriceExportBatch extends BaseBatch{
row << dist.name
row << o.s_date
def zgf_price = getLastThirdPrice(o.zhugezhaofang_esf_price, curPrice)
def zgf_price = getLastThirdPrice(o.xingfuli_esf_price, curPrice)
row << zgf_price
if(zgf_price){
add = true;
@@ -217,8 +217,8 @@ class ThirdPriceExportBatch extends BaseBatch{
if(!retMap.s_date){
retMap.s_date = curDate
}
if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){
retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price
if(!retMap.xingfuli_esf_price && d.xingfuli_esf_price){
retMap.xingfuli_esf_price = d.xingfuli_esf_price
}
if(!retMap.anjuke_esf_price && d.get("anjuke_esf_price")){
retMap.anjuke_esf_price = d.get("anjuke_esf_price")
@@ -3,6 +3,8 @@ package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
@@ -25,7 +27,30 @@ class AddAvgpriceBatch {
static main(args) {
def task = new AddAvgpriceBatch();
// task.run1();
task.test()
// task.test()
// task.sendPriceCommand()
task.sendNewDistPrice()
}
def sendNewDistPrice(){
DBObject q = new BasicDBObject();
q.put("status",0)
q.put("cat", "district");
q.put("c_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-11-14","yyyy-MM-dd")))
def dbsour = DbUtils.seaweed.find(q)
for(def d : dbsour){
addPrice(d.city,d.region,d.name, d.estate_type2, DateUtils.parseFromTightByPattern('2023-07-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-11-19','yyyy-MM-dd'))
}
}
def sendPriceCommand(){
ExcelUtils.multiThreadingModify("D:\\数据\\美术宝\\美术宝测试数据(价格).xlsx", { row ->
if(!row['P'] && row['T'] && ["别墅","普通住宅","商住两用"].contains(row['T'])){
//发重算均价命令
println row['K']+" "+row['L']+" "+row['M']
addPrice(row['K'],row['L'],row['M'], row['T'], DateUtils.parseFromTightByPattern('2023-10-01','yyyy-MM-dd'),DateUtils.parseFromTightByPattern('2023-10-25','yyyy-MM-dd'))
}
})
}
def test(){
@@ -1,5 +1,6 @@
package com.fangjia.script.gj
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -18,25 +19,24 @@ class EsSearchUtils {
static def findPois(def city, def region, def name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and adname = '" + region + "'")
sqlBuffer.append(" and name = '" + name + "'")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def sql = "select * from amap_poi where cityname='${city}' and adname='${region}' and MATCH(name, '${name}') "
ResultSet rs = dataSource.connection.createStatement().executeQuery(sql.toString())
def rets = []
while (rs.next()) {
def m = [:]
m.city = rs.getString("cityname")
m.region = rs.getString("adname")
m.name = rs.getString("name")
m.address = rs.getString("address")
m.type = rs.getString("type")
m.code = rs.getString("typecode")
m.lng = rs.getString("location.lon")
m.lat = rs.getString("location.lat")
rets << m
}
ElasticUtils.executeSqlQuery(sql,{ data->
def m = [:]
m.city = data.cityname
m.region = data.adname
m.name = data.name
m.address = data.address
m.type = data.type
m.code = data.typecode
def location = data.location
m.lng = location.lon
m.lat = location.lat
rets << m
})
return rets;
}
@@ -0,0 +1,53 @@
package com.fangjia.script.gj
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
DBCollection DISTRICT_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_sell_stat", "mongo.db.fangjia.com")
ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
File file = new File("d:\\Êý¾Ý\\ÃÀÍÅ\\FJW_COMM_20230815.csv");
def rows = [];
file.eachLine{
def d = it.split("@",-1)
if(d[1] && d[1]!='¸£ÖÝ' && d[7] && d[7].isNumber()){
rows << d
}
}
println rows.size()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rows.size())
for(def i=0;i<rows.size();i++){
final def d = rows[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def q = new BasicDBObject()
q.put("city", d[1])
q.put("region", d[2])
q.put("district_name", d[3])
q.put("date", 202307)
def udata = new BasicDBObject()
udata.put("avg_price", d[7].toString().toDouble().toInteger())
udata.put("m_date", new Date())
DISTRICT_SELL_STAT.update(q, new BasicDBObject('$set', udata), false, true)
println totalCount.get() +" "+d[1] + " "+d[3]
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
@@ -6,6 +6,7 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.CNNMFilter
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.gj.village.BaiduCoordinateConverter
import com.fangjia.script.util.CosineSimilarAlgorithm
import net.sf.json.JSONObject
import net.sf.json.groovy.JsonSlurper
@@ -337,7 +338,7 @@ class OpenApiUtils {
lat2 = null
lng2 = null
}
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+" "+map.estateType+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】"
}
// }else{
@@ -373,15 +374,18 @@ class OpenApiUtils {
}
static def searchBaiDuMap(String address, String region, int cityCode) {
static def searchBaiDuMap(String region, String township, String keyword,int cityCode) {
def rets = []
keyword = keyword.replace("居民委员会","").replace("居委会","").replace("居委会","").replace("居委","").replace("村民委员会","").replace("村村委会","村").replace("村委会","村")
def address = keyword
try {
Thread.sleep(500)
long t = System.currentTimeMillis()
// String url = "http://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1" +
// "&qt=con&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&sug=0&l=13" +
// "&sug_forward=&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}"
String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
// String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&c=${cityCode}&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
String url = "https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap&da_par=baidu&pcevaname=pc4.1&qt=s&da_src=searchBox.button&wd="+URLEncoder.encode(address,"UTF-8")+"&src=0&wd2=&pn=0&sug=0&l=11&from=webmap&sug_forward=&auth=7dDde8w2f6QfKBaV88xOONTKIxBMTxbHuxHNLELRBNLt1qo6DF%3D%3DC1GgvPUDZYOYIZuVt1cv3uVtGccZcuVtPWv3GuztQZ3wWvUvhgMZSguxzBEHLNRTVtcEWe1GD8zv7u%40ZPuVteuztghxehwzJP4G46zzWvnrRR1aDcvY1SGpuxEtEjjg2e&device_ratio=1&tn=B_NORMAL_MAP&nn=0&u_loc=13509900,3640104&ie=utf-8&t=${t}";
def content = new HttpRequest().get(url, null, "utf-8")
JSONObject json = JSONObject.fromObject(content)
@@ -389,49 +393,58 @@ class OpenApiUtils {
for (def loc : json.content) {
String name = formatName(address)
if(loc.uid) {
def bean = [:]
JSONObject uidInfo = BaiDuApiUtils.getDetailByUid(loc.uid)
JSONObject uidInfo = null;//BaiDuApiUtils.getDetailByUid(loc.uid)
if(uidInfo != null) {
def baiduRegion = uidInfo.get("result").area
def baiduCity = uidInfo.get("result").city
rets = createData(rets, uidInfo, loc, region, name, address)
}else{
def baiduCity = loc.city_name
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
def newName = formatName(loc.name)
if(baiduRegion.contains(region) && (name== newName || address==newName)){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
Double lng2 = lo.get("lng")
Double lat2 = lo.get("lat")
bean.lng2 = lng2
bean.lat2 = lat2
if(!baiduCity){
baiduCity = loc.history_address
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
}
if(!baiduCity){
baiduCity = loc.area_name
if(baiduCity.contains("市")){
baiduCity = baiduCity.substring(0, baiduCity.indexOf("市"))
}
baiduCity = RegionData.formatCity(baiduCity)
}
def baiduRegion = loc.area_name.replace(baiduCity,"")
keyword = keyword.replace("社区","")
def bdName = loc.name.replace("居民委员会","").replace("居委会","").replace("居委","").replace("村民委员会","").replace("村村委会","村").replace("村委会","村").replace("社区","")
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(bdName, keyword);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, keyword);
if(score1 || score2){
def bean = [:]
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
def lng = gmap.get("lng")
def lat = gmap.get("lat")
bean.lng = lng
bean.lat = lat
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
if(loc.admin_info){
bean.township = loc.admin_info.town_name
}
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
}else{
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
rets << bean;
def locArrs = BaiduCoordinateConverter.bdMercator2Latlng(loc.diPointX/100.0, loc.diPointY/100.0)
if(locArrs){
def gmap = GeocodeUtils.geoconvBaiduToGoogle(locArrs[0], locArrs[1])
bean.lng = gmap.get("lng")
bean.lat = gmap.get("lat")
}
bean.estateType = loc.std_tag
bean.score = score1 + score2
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+loc.name+"-"+ loc.addr.replace("[]","")+" "+bean.estateType+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
if(score1 || score2){
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
}
}
@@ -444,6 +457,71 @@ class OpenApiUtils {
return rets;
}
def static createData(def rets, def uidInfo, def loc, region, name, address){
def bean = [:]
def baiduRegion = uidInfo.get("result").area
def baiduCity = uidInfo.get("result").city
baiduCity = RegionData.formatCity(baiduCity)
def newName = formatName(loc.name)
if(baiduRegion.contains(region) && (name== newName || address==newName)){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
Double lng2 = lo.get("lng")
Double lat2 = lo.get("lat")
bean.lng2 = lng2
bean.lat2 = lat2
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
bean.township = loc.admin_info.town_name
bean.score = 1
def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2)
def lng = gmap.get("lng")
def lat = gmap.get("lat")
bean.lng = lng
bean.lat = lat
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
}
if(bean.estateType && (bean.estateType.contains('出入口') || bean.estateType.contains('公交车站'))){
println "X百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】-${bean.estateType}"
}else{
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+result.get("address")+"】" +" 搜索:【"+address+"】"
rets << bean;
}
}else{
def score1 = CosineSimilarAlgorithm.getSimilarityRatio(loc.name, address);
def score2 = CosineSimilarAlgorithm.getSimilarityRatio(loc.addr, address);
if(score1 || score2){
JSONObject result = uidInfo.get("result")
JSONObject lo = result.get("location")
bean.lng2 = lo.get("lng")
bean.lat2 = lo.get("lat")
bean.name = loc.name
bean.address = loc.addr.replace("[]","")
bean.city = baiduCity
bean.region = baiduRegion
bean.township = loc.admin_info.town_name
def gmap = GeocodeUtils.geoconvBaiduToGoogle(bean.lng2, bean.lat2)
bean.lng = gmap.get("lng")
bean.lat = gmap.get("lat")
if(result.get("detail_info")){
bean.estateType = result.get("detail_info").get("tag")
}
bean.score = score1 + score2
rets << bean;
println "√百度搜索地图:【"+baiduCity+"-"+baiduRegion+"-"+newName+"-"+bean.estateType+"-"+ loc.addr.replace("[]","")+"】" +" 搜索:【"+address+"】(score:"+score1+"#"+score2+")"
}else{
println "X 百度搜索地图:【"+loc.get("area_name")+"-"+loc.get("area_name")+"-"+loc.get("name")+"-"+loc.get("addr")+"】" +" 搜索:【"+address+"】"
}
}
return rets;
}
def static formatName(String name) {
if(name.indexOf("¡¤") != -1) {
name = name.replaceAll("¡¤", "")
@@ -566,9 +644,70 @@ class OpenApiUtils {
return null;
}
static def jsHouseCase(city, region, name, propertyTypeCode){
def url = "http://localhost:8080/fangjia-open/house/case?city=${city}&district=${region}&buildingName=${name}&propertyTypeCode=${propertyTypeCode}"
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def jsHouseCaseByAddress(address, area){
address = address.replace(" ","").replace("%","")
// def url = "http://open.fangjia.com/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
def url = "http://localhost:8080/fangjia-open/house/case?address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(area){
url = url +"&strctureArea="+area
}
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def houseValueInformation(city, address, area){
address = address.replace(" ","").replace("%","")
def url = "http://open.fangjia.com/comprehensiveQuery/houseValueInformation?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
if(area){
url = url +"&size="+area
}
// println url
String content = new HttpRequest().get(url, "utf-8");
def dto = []
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dtos = doc.result
return dtos
}else{
println url
}
}
return null;
}
static def financialRating(city, address){
def url = "http://open.fangjia.com/address/financialRating?city=${city}&address=${address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
String content = new HttpRequest().get(url, "utf-8");
print(url)
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==0 && doc.result){
@@ -158,7 +158,6 @@ class PoiLocation {
gmap.lat = DataHandleUtils.getDoubleRound(6, gmap.lat)
}
return location;
}
return null;
}
@@ -48,11 +48,11 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2023-06-01"
static def statDate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1),"yyyy-MM-dd")
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
// task.esfParser("D:\\数据\\友达\\2023年7月\\查询1.DBF");//友达二手房存入mongo
// task.esfParser("D:\\数据\\友达\\2024年2月\\查询1.DBF");//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
@@ -0,0 +1,372 @@
package com.fangjia.script.gj.abc
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class SuzouPocTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new SuzouPocTest()
// task.run()
task.newRun()
}
def newRun(){
println("start:"+new Date().toString())
def filePath = "D:\\数据\\江苏农行\\押品填报表_测试数据样例-给公司.xls"
newMatch(filePath)
addEvalutePrice(filePath)
addLation(filePath)
addAroundPrice(filePath)
println("end:"+new Date().toString())
}
def newMatch(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
if(d['G'] && !d['J']){
def city = d['B']
def address = d['E']
def name = d['G']
def region = d['C']
if(region && name){
address = region + name
}
city = RegionData.formatCity(city)
if(region){
region = RegionData.formatRegion(city, region)
}
def obj = CommonDatautils.findDistrict(city, region, name.trim(), "district");
if(obj){
d['M'] = obj._id.toString()
d['N'] = obj.get("city")
d['O'] = obj.get("region")
d['P'] = obj.get("name")
if(obj.lng){
d['Q'] = obj.get("lng")/10000000000.0
d['R'] = obj.get("lat")/10000000000.0
}
d['S'] = "直接从库中查询"
}else{
//新的匹配算法
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(!jsonObject && !jsonObject.containsKey("result")){
jsonObject = FangjiaClientUtils.newMatch(city, address, "property", true)
}
if(!jsonObject && !jsonObject.containsKey("result")){
def newAddr = d['E']
if(!d['E'].contains(d['C'])){
newAddr = d['C'] + d['E']
}
jsonObject = FangjiaClientUtils.newMatch(city, newAddr, "property", true)
}
if(!jsonObject.containsKey("result")){
//老匹配
jsonObject = OpenApiClient.addressMatch(city, address, null, false, region, name)
if(jsonObject && jsonObject.credit>0){
d['S'] = "老匹配"
}
}
if (jsonObject != null && !jsonObject.containsKey("result")) {
d['M'] = jsonObject.get("id")
d['N'] = jsonObject.get("city")
d['O'] = jsonObject.get("district")
d['P'] = jsonObject.get("name")
d['Q'] = jsonObject.get("lng")
d['R'] = jsonObject.get("lat")
}
}
}
});
}
def addEvalutePrice(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
def city = d['N']
if(city && !d['J']){
def region = d['O']
def name = d['P']
def area = d['F']
def address = d['E']
def floor = null
if (d['H'] && d['H'].toString().isNumber()) {
floor = d['H'].toString().toDouble().toInteger().toString()
}
def height = null
if (d['J'] && d['J'].toString().isNumber()) {
height = d['J'].toString().toDouble().toInteger().toString()
}
def ret1 = OpenApiClient.addressEvaluation(city, region, name, name, area, floor, height)
if (ret1) {
d['J'] = ret1.totalPrice
d['K'] = ret1.avgPrice
}else{
def priceObj = CommonDatautils.findDistrictPrice(city, region, name)
if(priceObj){
d['K'] = priceObj.avg_price_2b
if(area && area.toString().isNumber() && !area.toString().contains("+")){
d['J'] = (int)(area.toString().toDouble()*priceObj.avg_price_2b)
}
}
}
}
})
}
def addLation(filePath){
ExcelUtils.multiThreadingModify(filePath, { d ->
def city = d['B']
if(city && !d['J']){
def address = d['E']
def map = parserByGaode(city, address)
if(map){
d['Q'] = map.lng
d['R'] = map.lat
d['T'] = "地址反解析经纬度"
}
}
})
}
def parserByGaode(def city, def address){
def json = GaoDeApiUtils.geo(city, address)
println json
if(json && json.status=="1"){
def ret = json.geocodes
if(ret){
def str = ret.get(0).location
def level = ret.get(0).level
if(str && ["门牌号","单元号","道路","道路交叉路口","热点商圈","村庄","兴趣点","门址","住宅区"].contains(level)){
def arrs = str.split(",")
return ["lng":arrs[0], "lat":arrs[1]]
}else{
println address +" "+level
return null
}
}
}
}
def addAroundPrice(path){
ExcelUtils.multiThreadingModify(path, { d ->
if(!d['J'] && d['Q']){
def city = RegionData.formatCity(d['B'])
def dist = new BasicDBObject()
dist.city = city
dist.lng = d['Q'].toString().toDouble()*10000000000L
dist.lat = d['R'].toString().toDouble()*10000000000L
dist.estateType = "普通住宅"//d['H']
if(dist.lat) {
List<DBObject> list = DbUtils.findAroundDistrictByRadius(dist, 1000)
if (list) {
list = sortByDistance(dist, list);
if (list && list.size() > 10) {
list = list[0..9]
}
if (list) {
def prices = []
for (def s : list) {
def priceObj = CommonDatautils.findDistrictPrice(s.city, s.region, s.name)
if (priceObj) {
prices << priceObj.avg_price_2b
}
}
if (prices) {
def arround_price = (int) (prices.sum() / prices.size())
d['k'] = arround_price
def area = d["F"]
if(area && area.toString().isNumber() && !area.toString().contains("+")){
d['J'] = (int)(area.toString().toDouble()*arround_price)
}
}
}
}
}
}
});
}
def sortByDistance(d, list){
def newList = []
for(def nb : list){
def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0)
nb.distance = distance
newList << nb
}
//排序
Collections.sort(newList, new Comparator() {
@Override
int compare(Object o1, Object o2) {
if(o1.distance > o2.distance)
return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1
if(o1.distance == o2.distance)
return 0;
return -1;
}
})
return newList;
}
def run(){
def path = "D:\\数据\\江苏农行\\江苏农1062条(总).xlsx"
addressEvalute(path)
// addFjwTobPrice(path)
// addThirdPrice(path)
// margeExcel(path)
}
def addressEvalute(path){
ExcelUtils.multiThreadingModify(path,{d->
if(!d['O'] && d['H']){
def city = d['D']
def region = d['E']
def address = d['G']
def name= d['H']
def area = d['I']
def floor = null
if( d['J'] && d['J'].toString().isNumber()){
floor = d['J'].toString().toDouble().toInteger().toString()
}
def height = null
if(d['K'] && d['K'].toString().isNumber()){
height = d['K'].toString().toDouble().toInteger().toString()
}
def ret1 = OpenApiClient.addressEvaluation(city, region, address, name, area, floor, height)
if(ret1){
d['O']= ret1.city
d['P'] = ret1.district
d['Q'] = ret1.name
d['R'] = ret1.totalPrice
d['S'] = ret1.avgPrice
}
def ret2 = OpenApiClient.addressEvaluation(city, region, address, name, area, null, null)
if(ret2){
d['T'] = ret2.totalPrice
d['U'] = ret2.avgPrice
}
}
})
}
def addFjwTobPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['O']){
def priceObj = CommonDatautils.findDistrictPrice(d['O'], d['P'], d['Q'])
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
}
})
}
def addThirdPrice(path) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { d ->
if (d['O'] && !d['V']) {
def dist = CommonDatautils.findDistrict(d['O'], d['P'], d['Q'], "district")
if (dist) {
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310)
def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name)
if (priceMap) {
if (priceMap.containsKey("lianjia_esf_price")) {
d['X'] = priceMap.get("lianjia_esf_price")
}
if (priceMap.containsKey("beike_esf_price")) {
d['Y'] = priceMap.get("beike_esf_price")
}
if (priceMap.containsKey("anjuke_esf_price")) {
d['Z'] = priceMap.get("anjuke_esf_price")
}
}
def cityId = cityCodeDataMap.get(dist.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
if (map) {
d['AA'] = map.avg_price
}
def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01')
if (tradePrice) {
d['AB'] = tradePrice.toInteger()
}
}
}
})
}
def margeExcel(path){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\(合并)江苏农匹配不上149条-.xlsx", 0, 1)
def map = [:]
for(def d : datas){
map.put(d['G'], d['O'])
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\江苏农1062条.xlsx",{d->
def commId = map.get(d['G'])
def dist = CommonDatautils.findDistrictById(commId)
if(dist){
d['O'] = dist.city
d['P'] = dist.region
d['Q'] = dist.name
def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
if(priceObj){
d['V'] = priceObj.avg_price
d['W'] = priceObj.avg_price_2b
}
}
})
}
}
@@ -230,12 +230,46 @@ class CcbCodeUtils {
}
return "11"
}
//室内结构
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
static def getHouseStructureCode(){
return null
}
//室内结构
//平层:11 错层:12 复式:13 阁楼:14 团结户:15 老式房:16 地下室:17 带阁楼:21 带车库:22 其他:99 跃层:18 带地下室:24 带储藏室:25 带车位:23
static def getRoomStructureCode(room_structure){
if(room_structure){
if(room_structure.contains("平层")){
return "11"
}else if(room_structure.contains("错层")){
return "12"
}else if(room_structure.contains("复式")){
return "13"
}else if(room_structure.contains("阁楼")){
return "14"
}else if(room_structure.contains("团结户")){
return "15"
}else if(room_structure.contains("老式房")){
return "16"
}else if(room_structure.contains("地下室")){
return "17"
}else if(room_structure.contains("带阁楼")){
return "21"
}else if(room_structure.contains("带车库")){
return "22"
}else if(room_structure.contains("带车位")){
return "23"
}else if(room_structure.contains("带地下室")){
return "24"
}else if(room_structure.contains("带储藏室")){
return "25"
}else if(room_structure.contains("跃层")){
return "18"
}
}
return null;
}
//租赁方式 整租:0 合租:1 转租:2
static def getRentTypeCode(def rentType){
if(rentType){
@@ -90,4 +90,53 @@ class FileUtils {
}
return ret;
}
static def batchCreateFileBySplitChar(ret, fileName, headers,charset, isAppend, splitChar){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
if(headers){
sb.append(headers.join(splitChar))
sb.append("\n");
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
def index = 0
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), splitChar))
sb.append("\n");
if(index%10000==0){
println index+"¿ªÊ¼Ð´ÈëCSV "+new Date().toLocaleString()
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
println index+"½áÊøÐ´ÈëCSV "+new Date().toLocaleString()
}
}
}
if(sb.length()>0){
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
}
out.close();
}
}
@@ -0,0 +1,317 @@
package com.fangjia.script.gj.chongqin
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
/**
* 重庆poc测试-2023版
*/
class PocTest2023 {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection paimai43_COLL = MongoUtils.getCollectionByName2("fangjia_crawl", "pai_mai", lineUrl)
public static final DBCollection DISTRICT_ANALY_DATA_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_analy_data", "mongo.db.fangjia.com")
static main(args) {
def task = new PocTest2023();
task.run();
}
def run(){
// addEvaluationPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试2.xlsx")
// addressEvalute("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx")
// structure("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx") //九级地址拆分
// addThirdPrice("D:\\数据\\重庆农商行2023\\重庆估值样本测试.xlsx")
// baseInfo("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx")
// baseInfoByName("D:\\数据\\重庆农商行2023\\开发商及物业信息样本.xlsx")
distScore("D:\\数据\\重庆农商行2023\\楼盘评级样本.xlsx")
}
def addEvaluationPrice(path){
ExcelUtils.multiThreadingModify(path,{ d->
if(d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['N']= ret1.city
d['O'] = ret1.district
d['P'] = ret1.name
d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0)
d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0)
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['Q'] = dist._id.toString()
}
}
}
})
}
def addressEvalute(path){
ExcelUtils.multiThreadingModify(path,{ d->
if(!d['N'] && d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['N']= ret1.city
d['O'] = ret1.district
d['P'] = ret1.name
d['F'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.avgPrice/10000.0)
d['G'] = com.fangjia.script.gj.paimai.DataUtil.getRound(2, ret1.totalPrice/10000.0)
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['Q'] = dist._id.toString()
}
}
//新的匹配算法
def jsonObject = FangjiaClientUtils.newMatch(city, address, "property", false)
if(jsonObject != null && jsonObject.credit>0) {
d['R'] = jsonObject.get("city")
d['S'] = jsonObject.get("district")
d['T'] = jsonObject.get("name")
d['U'] = jsonObject.get("id")
}
if( d['Q'] && d['U'] && d['U']==d['Q']){
d['V'] = "T"
}else{
d['V'] = "F"
}
}
})
}
def structure(inputfile){
ExcelUtils.multiThreadingModify(inputfile,{ d->
if(d['E']){
def jsonObject = FangjiaClientUtils.newStructureByLevel9(d['B'], d['E'])
if(jsonObject != null && jsonObject.msgCode==200 && jsonObject.result) {
def ret = jsonObject.result
def str = ""
if(ret.township){
str = ret.township
}
if(ret.street){
str = str + ret.street
}
d['H'] = str
d['I'] = ret.name
d['J'] = ret.buildingNumber
d['K'] = ret.unitNumber
d['L'] = ret.floor
d['M'] = ret.roomNumber
}
}
})
}
// def addAuthor(inputfile){
// def authors = ["曹","张", "陈", "许", "金玉"]
// ExcelUtils.modify(inputfile, { d ->
// if(d['V'] && d['V'] == "F"){
// for(def name : authors){
// d['W'] = name
// }
// }
// });
// }
def baseInfo(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { d ->
if(d['E']){
def city = d['B']
def region = d['C']
def address = d['E']
def area = d['D']
println address
//旧地址估值
def ret1 = OpenApiClient.addressEvaluation(city, region, address, null, area, null, null)
if(ret1){
d['R']= ret1.city
d['S'] = ret1.district
d['F'] = ret1.name
def dist = CommonDatautils.findDistrict(ret1.city, ret1.district, ret1.name, "district")
if(dist){
d['T'] = dist._id.toString()
d['G'] = dist.estate_company
d['H'] = dist.developer
d['I'] = dist.cubage_percent
d['J'] = dist.virescence_percent
d['K'] = dist.all_area
d['L'] = dist.loudong_count
d['M'] = dist.household_count
d['N'] = dist.estate_charge
if(dist.open_date){
d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd")
}
if(dist.is_elevator){
d['P'] = "有"
}else{
d['P'] = "无"
}
def newCity = RegionData.formatCity(city)
def obj = findPaiMai(newCity, address)
if(obj){
d['Q'] = "是"
}
}
}
}
})
}
def findPaiMai(city, address){
def q = new BasicDBObject()
q.put("city", city)
def or = new BasicDBList()
or.add(new BasicDBObject("title", new BasicDBObject('$regex', address)))
or.add(new BasicDBObject("address", new BasicDBObject('$regex', address)))
q.append('$or', or)
def obj = paimai43_COLL.findOne(q)
return obj
}
def baseInfoByName(inputfile){
ExcelUtils.multiThreadingModify(inputfile, { d ->
if(d['F']) {
def city = d['B']
def address = d['E']
def dist = CommonDatautils.findDistrict(d['R'], d['S'], d['F'], "district")
if (dist) {
d['T'] = dist._id.toString()
d['G'] = dist.estate_company
d['H'] = dist.developer
d['I'] = dist.cubage_percent
d['J'] = dist.virescence_percent
d['K'] = dist.all_area
d['L'] = dist.loudong_count
d['M'] = dist.household_count
d['N'] = dist.estate_charge
if (dist.open_date) {
d['O'] = DateUtils.formatDateByPattern(dist.open_date, "yyyy-MM-dd")
}
if (dist.is_elevator) {
d['P'] = "有"
} else {
d['P'] = "无"
}
def newCity = RegionData.formatCity(city)
def obj = findPaiMai(newCity, address)
if (obj) {
d['Q'] = "是"
}
}
}
})
}
def distScore(inputfile) {
ExcelUtils.multiThreadingModify(inputfile, { d ->
if (d['F']) {
def obj = getDistrictRank(d['R'], d['S'], d['F'])
if(obj){
def score = obj.city_rank*100/obj.city_rank_total
score = DataUtil.getRound(score)
d['G'] = 100-score
d['H'] = obj.city_rank
d['I'] = obj.city_rank_total
d['J'] = obj.avg_price
}
}
})
}
def getDistrictRank(def city, def region, def name){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name",name)
return DISTRICT_ANALY_DATA_COLL.findOne(q)
}
def addThirdPrice(path) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
ExcelUtils.multiThreadingModify(path, { d ->
if (d['P']) {
def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
if (dist) {
// def priceObj = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name)
// if(priceObj){
// d['V'] = priceObj.avg_price
// d['W'] = priceObj.avg_price_2b
// }
def cityMap = CommonDatautils.findThirdPartyPrice(dist.city, dist.region, dist.name, 202303, 202310)
def priceMap = cityMap.get(dist.city + "##" + dist.region + "##" + dist.name)
if (priceMap) {
if (priceMap.containsKey("lianjia_esf_price")) {
d['X'] = priceMap.get("lianjia_esf_price")
}
if (priceMap.containsKey("beike_esf_price")) {
d['Y'] = priceMap.get("beike_esf_price")
}
if (priceMap.containsKey("anjuke_esf_price")) {
d['Z'] = priceMap.get("anjuke_esf_price")
}
}
def cityId = cityCodeDataMap.get(dist.city)
def map = CcbCommonUtils.findAvgPrice(dist._id.toString(), cityId)
if (map) {
d['AA'] = map.avg_price
}
def tradePrice = CommonDatautils.findTrades(dist, '2023-03-01', '2023-10-01')
if (tradePrice) {
d['AB'] = tradePrice.toInteger()
}
}
}
})
}
}
@@ -1,19 +1,19 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.city
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
class CityStatExport {
@@ -28,8 +28,41 @@ class CityStatExport {
def task = new CityStatExport();
// task.run()
// task.run1()
task.regionStat()
// task.regionStat()
// task.stat()
// task.houseStat()
task.addFieldStat()
}
def addFieldStat(){
def fileName = "D:\\数据\\fjw.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
if(!d['B']){
def city = RegionData.formatCity(d['A'])
if(city){
def cityInfoDto = CommonDatautils.findCityInfo(city)
if(cityInfoDto){
d['B'] = cityInfoDto.existing_district_count
def detailMap = cityInfoDto.district_detail
// d['D'] = detailMap.address.toDouble()*100
// d['E'] = 100
d['D'] = cityInfoDto.price_coverage.toDouble()*100
}
}
}
},1)
}
def houseStat(){
def fileName = "D:\\数据\\贝壳\\贝壳城市小区二手房统计V3-已重新补充.xlsx"
ExcelUtils.multiThreadingModify(fileName, { d ->
// def houseCount = findHouseCount(d['F'], 20230201)
// d['H'] = houseCount
d['J'] = findHouse(d['F'])
})
}
def stat(){
@@ -47,7 +80,7 @@ class CityStatExport {
// def citys = CommonDatautils.findAllCity();
def citys = ["杭州","宁波","温州","嘉兴","台州","湖州","绍兴","金华","舟山","衢州","丽水"]
def citys = [ "贵阳","六盘水","遵义","安顺","铜仁","毕节","黔西南","黔东南","黔南"]
def ret = []
ret << ["城市","小区总量"]
// ret << ["城市","楼栋总量", "房号总量"]
@@ -78,7 +111,7 @@ class CityStatExport {
ret << m
println m
}
ExcelUtils.write("D:\\数据\\浙江省城市统计(楼盘量).xlsx",ret,"数据")
ExcelUtils.write("D:\\数据\\楼盘量.xlsx",ret,"数据3")
}
@@ -124,7 +157,7 @@ class CityStatExport {
def findHouseCount(def city, def startDate){
def total = 0
try{
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
String sql = "select count(*) as c from seawater where cat='sell' and city='${city}' and p_date >= ${startDate} "
ElasticUtils.executeSqlQuery(sql,{ data->
total = data.c
@@ -136,6 +169,18 @@ class CityStatExport {
return total
}
def findHouse(city) {
int startDate = 20230201
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("cat", "sell"))
.must(date_range)
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses.size()
}
def findLoudongCount(String city) {
return LOUDONG.count(new BasicDBObject('city', city))
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.commStat
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.CcbCodeUtils
@@ -27,7 +28,7 @@ ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
DBCollection region_stat_coll = MongoUtils.getCollectionByName("fangjia", "region_stat", "192.168.0.43")
DBCollection mca_administrative_division_coll = MongoUtils.getCollectionByName("crawl", "mca_administrative_division_2022", "192.168.0.206", 27020)
//def citys = ["乌鲁木齐","昌吉","阿克苏","伊犁","喀什"]
//for(def city : citys){
@@ -308,21 +309,65 @@ def findHousesByCityAndRoot(def city, def cat, def startDate, def root){
//ExcelUtils.write("D:\\数据\\江苏\\新疆数据统计.xlsx", rows, "结果")
//区域级统计
def city = "合肥"
def regions = CommonDatautils.getCityRegions(city)
def rows = []
for(def region : regions){
def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
def alias_count = 0
def addr_alias_count = 0
for(def r : districts){
if(r.alias && r.alias.size()>0){
alias_count ++
//def city = "合肥"
//def regions = CommonDatautils.getCityRegions(city)
//def rows = []
//for(def region : regions){
// def districts = CommonDatautils.findDistrictByCityAndRegions(city, "district", [region], ["city","region","name","alias", "addr_alias"])
// def alias_count = 0
// def addr_alias_count = 0
// for(def r : districts){
// if(r.alias && r.alias.size()>0){
// alias_count ++
// }
// if(r.addr_alias && r.addr_alias.size()>0){
// addr_alias_count++
// }
// }
// rows << [region, districts.size(), alias_count, addr_alias_count]
//}
//ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
ExcelUtils.multiThreadingModify("D:\\数据\\临时\\区县楼盘量统计.xlsx",{d->
if(d['C'] && !d['I']){
def city = RegionData.formatCity(d['C'])
if(!city){
city = RegionData.formatCity(d['B'])
}
if(r.addr_alias && r.addr_alias.size()>0){
addr_alias_count++
if(city && d['D']){
def region = RegionData.formatRegion(city, d['D'])
if(region){
def count = CommonDatautils.findRegionDistrictCount("district", city, region)
println city +" "+ region+" "+count
d['I'] = city
d['J'] = region
d['K'] = count
}
// def dists = CommonDatautils.findAllDistricts(d['C'], "district", ["city","region","name"])
//d['B'] = dists.size()
}
}
rows << [region, districts.size(), alias_count, addr_alias_count]
}
ExcelUtils.write("D:\\数据\\新安银行\\新安银行数据统计.xlsx", rows, "结果")
// def q = new BasicDBObject()
// q.put("city",d['B'])
// q.put("district","")
// def dbo = mca_administrative_division_coll.findOne(q)
// if(!dbo){
// q.put("city", "省直辖县级行政区划")
// q.put("district", d['B'])
// dbo = mca_administrative_division_coll.findOne(q)
// }
// if(dbo){
// d['A']=dbo.province
// }
})
//def citys = ["北京","上海","重庆","天津","石家庄","唐山","秦皇岛","邯郸","邢台","保定","张家口","承德","沧州","廊坊","衡水","太原","大同","阳泉","长治","晋城","朔州","晋中","运城","忻州","临汾","吕梁","呼和浩特","包头","乌海","赤峰","通辽","鄂尔多斯","呼伦贝尔","巴彦淖尔","乌兰察布","兴安盟","锡林郭勒盟","阿拉善盟","沈阳","大连","鞍山","抚顺","本溪","丹东","锦州","营口","阜新","辽阳","盘锦","铁岭","朝阳","葫芦岛","长春","吉林","四平","辽源","通化","白山","松原","白城","延边","哈尔滨","齐齐哈尔","鸡西","鹤岗","双鸭山","大庆","伊春","佳木斯","七台河","牡丹江","黑河","绥化","大兴安岭","南京","无锡","徐州","常州","苏州","南通","连云港","淮安","盐城","扬州","镇江","泰州","宿迁","杭州","宁波","温州","嘉兴","湖州","绍兴","金华","衢州","舟山","台州","丽水","合肥","芜湖","蚌埠","淮南","马鞍山","淮北","铜陵","安庆","黄山","滁州","阜阳","宿州","六安","亳州","池州","宣城","福州","厦门","莆田","三明","泉州","漳州","南平","龙岩","宁德","南昌","景德镇","萍乡","九江","新余","鹰潭","赣州","吉安","宜春","抚州","上饶","济南","青岛","淄博","枣庄","东营","烟台","潍坊","济宁","泰安","威海","日照","临沂","德州","聊城","滨州","菏泽","郑州","开封","洛阳","平顶山","安阳","鹤壁","新乡","焦作","濮阳","许昌","漯河","三门峡","南阳","商丘","信阳","周口","驻马店","武汉","黄石","十堰","宜昌","襄阳","鄂州","荆门","孝感","荆州","黄冈","咸宁","随州","恩施","长沙","株洲","湘潭","衡阳","邵阳","岳阳","常德","张家界","益阳","郴州","永州","怀化","娄底","湘西","广州","韶关","深圳","珠海","汕头","佛山","江门","湛江","茂名","肇庆","惠州","梅州","汕尾","河源","阳江","清远","东莞","中山","潮州","揭阳","云浮","南宁","柳州","桂林","梧州","北海","防城港","钦州","贵港","玉林","百色","贺州","河池","来宾","崇左","海口","三亚","三沙","儋州","成都","自贡","攀枝花","泸州","德阳","绵阳","广元","遂宁","内江","乐山","南充","眉山","宜宾","广安","达州","雅安","巴中","资阳","阿坝州","甘孜州","凉山州","贵阳","六盘水","遵义","安顺","毕节","铜仁","黔西南","黔东南","黔南","昆明","曲靖","玉溪","保山","昭通","丽江","普洱","临沧","楚雄","红河","文山州","西双版纳州","大理","德宏州","怒江州","迪庆州","拉萨","日喀则","昌都","林芝","山南","那曲","阿里","西安","铜川","宝鸡","咸阳","渭南","延安","汉中","榆林","安康","商洛","兰州","嘉峪关","金昌","白银","天水","武威","张掖","平凉","酒泉","庆阳","定西","陇南","临夏","甘南","西宁","海东","海北州","黄南州","海南州","果洛州","玉树州","海西州","银川","石嘴山","吴忠","固原","中卫","乌鲁木齐","克拉玛依","吐鲁番","哈密","昌吉","博尔塔拉","巴音郭楞","阿克苏","克孜勒苏","喀什","和田","伊犁","塔城","阿勒泰","济源","仙桃","潜江","天门","神农架","五指山","琼海","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中","石河子","阿拉尔","图木舒克","五家渠","北屯","铁门关","双河","可克达拉","昆玉"]
//
//def fjwCitys = CommonDatautils.findAllCity()
//for(def fjw_city : fjwCitys){
// if(!citys.contains(fjw_city)){
// println fjw_city
// }
//}
@@ -14,12 +14,12 @@ class FieldStat {
static main(args) {
def task = new FieldStat();
// task.run()
// task.run1()
task.run1()
// task.run2()
// task.run3()
// task.offircRun()
task.distRun()
task.shopRun()
// task.distRun()
// task.shopRun()
}
def distRun(){
@@ -203,11 +203,15 @@ class FieldStat {
alias_total = alias_total + alias.size()
}
}
def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
def precent = 0
if(districts){
precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size())
}
rets << [city, alias_total, alias_index, districts.size(), precent+"%"]
}
ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计")
ExcelUtils.write("d:\\数据\\城市别名统计.xlsx", rets, "统计")
// println dist_total
// println alias_total
// println addr_alias_total
@@ -5,24 +5,38 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//
//ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
// def opt = d['J']
// if(opt && opt.contains("别名")){
// def dist = CommonDatautils.findDistrictById(d['H'])
// if(dist){
// def alias = []
// if(dist.alias){
// alias = dist.alias
// }
// if(!alias.contains(d['C'])){
// alias << d['C']
// def udata = new BasicDBObject()
// udata.put("alias", alias)
// DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
// }
//
// }
// }
//})
ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d->
ExcelUtils.modify("D:\\数据\\厦门\\厦门户数(曹和付)-结果.xlsx",{d->
def opt = d['J']
if(opt && opt.contains("别名")){
def dist = CommonDatautils.findDistrictById(d['H'])
if(opt && opt.toString().toDouble().toInteger()==1){
def dist = CommonDatautils.findDistrict('厦门',d['G'],d['H'], "district")
if(dist){
def alias = []
if(dist.alias){
alias = dist.alias
}
if(!alias.contains(d['C'])){
alias << d['C']
def udata = new BasicDBObject()
udata.put("alias", alias)
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
def udata = new BasicDBObject()
udata.put("household_count", d['F'].toString().toDouble().toInteger())
DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata))
}
}
})
@@ -5,8 +5,10 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
@@ -22,12 +24,177 @@ class ApiTest {
static main(args) {
def task = new ApiTest()
// task.rankRunTest()
// task.similarRunTest()
task.loupanLevelTest()
// task.rankRunTest()//小区排名接口测试
// task.similarRunTest()//相似房源接口测试
// task.loupanLevelTest()//楼盘评级接口测试
// task.jsAbcHouseCastTest()//北交所房源接口测试 小区
// task.arroundFacilities()//北交所周边poi 小区
// task.addressAvgPrice("D:\\数据\\北交所\\拍卖案例(-地址估值接口).xlsx")//地址出小区均价
// task.addressEvaluate("D:\\数据\\北交所\\拍卖案例(-地址均价接口).xlsx")//地址估值
// task.arroundFacilitiesByAddress("D:\\数据\\北交所\\拍卖案例(-周边poi接口).xlsx")//北交所周边poi 地址
// task.jsAbcHouseCastTestByAddress("D:\\数据\\北交所\\拍卖案例(房源接口500条-价格控20%).xlsx")
task.houseValueInformation("D:\\数据\\北交所\\北交所综合接口测试.xlsx")
//五级地址标准化
// task.addrStructure("D:\\数据\\地数\\地址标准化-测试用例.xlsx")
// task.nineLevel("D:\\数据\\地数\\地址九级标准化-测试用例.xlsx")
}
def addrStructure(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
JSONObject jsonObject = FangjiaClientUtils.formatFivelevel(d['A'], d['B']);
if(jsonObject) {
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['C'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['D'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['E'] = jsonObject.get("district")
}
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
d['F'] = jsonObject.get("township")
}
if(jsonObject.get("address") && jsonObject.get("address")!="null"){
d['G'] = jsonObject.get("address")
}
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
d['H'] = jsonObject.get("provinceCode")
}
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
d['I'] = jsonObject.get("cityCode")
}
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
d['J'] = jsonObject.get("districtCode")
}
}
}
},7);
}
def nineLevel(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def obj = FangjiaClientUtils.formatNinelevel(d['A'], d['B']);
if(obj) {
def jsonObject = obj.get("result")
if(jsonObject){
if(jsonObject.get("province") && jsonObject.get("province")!="null"){
d['C'] = jsonObject.get("province")
}
if(jsonObject.get("city") && jsonObject.get("city")!="null"){
d['D'] = jsonObject.get("city")
}
if(jsonObject.get("district") && jsonObject.get("district")!="null"){
d['E'] = jsonObject.get("district")
}
if(jsonObject.get("township") && jsonObject.get("township")!="null"){
d['F'] = jsonObject.get("township")
}
if(jsonObject.get("formatAddress") && jsonObject.get("formatAddress")!="null"){
d['G'] = jsonObject.get("formatAddress")
}
if(jsonObject.get("community") && jsonObject.get("community")!="null"){
d['H'] = jsonObject.get("community")
}
if(jsonObject.get("name") && jsonObject.get("name")!="null"){
d['I'] = jsonObject.get("name")
}
if(jsonObject.get("street") && jsonObject.get("street")!="null"){
d['J'] = jsonObject.get("street")
}
if(jsonObject.get("streetNumber") && jsonObject.get("streetNumber")!="null"){
d['K'] = jsonObject.get("streetNumber")
}
if(jsonObject.get("buildingNumber") && jsonObject.get("buildingNumber")!="null"){
d['L'] = jsonObject.get("buildingNumber")
}
if(jsonObject.get("unitNumber") && jsonObject.get("unitNumber")!="null"){
d['M'] = jsonObject.get("unitNumber")
}
if(jsonObject.get("roomNumber") && jsonObject.get("roomNumber")!="null"){
d['N'] = jsonObject.get("roomNumber")
}
if(jsonObject.get("floor") && jsonObject.get("floor")!="null"){
d['O'] = jsonObject.get("floor")
}
if(jsonObject.get("provinceCode") && jsonObject.get("provinceCode")!="null"){
d['P'] = jsonObject.get("provinceCode")
}
if(jsonObject.get("cityCode") && jsonObject.get("cityCode")!="null"){
d['Q'] = jsonObject.get("cityCode")
}
if(jsonObject.get("districtCode") && jsonObject.get("districtCode")!="null"){
d['R'] = jsonObject.get("districtCode")
}
def map = jsonObject.get("addressAssessment")
if(map){
d['S'] = map.get("msg")
}
}
}
}
},0);
}
def addressEvaluate(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def ret1 = OpenApiClient.addressEvaluation(d['A'], null, d['B'], null, d['C'].toString().toDouble().toString(), null, null)
if(ret1){
d['G']= ret1.city
d['H'] = ret1.district
d['I'] = ret1.name
d['J'] = ret1.totalPrice
d['K'] = ret1.avgPrice
}
}
})
}
def addressAvgPrice(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
println "[match] city:"+d['A']+" address:"+d['B']
def jsonObject = FangjiaClientUtils.newMatchPrice(d['A'], d['B'], "property", false)
if(jsonObject && jsonObject.containsKey("result")) {
def resultMap = jsonObject.get("result")
d['G'] = resultMap.get("city")
d['H'] = resultMap.get("district")
d['I'] = resultMap.get("name")
def type = resultMap.get("propertyType")
if(type && type!='null'){
d['J'] = type
}
if(resultMap.get("avgPrice")){
d['K'] = resultMap.get("avgPrice")
}
}
}
})
}
def loupanLevelTest(){
ExcelUtils.modify("D:\\数据\\卓信\\上海 - 副本.xlsx", { d ->
if (d['A']) {
@@ -125,6 +292,265 @@ class ApiTest {
return dbo
}
def jsAbcHouseCastTest(){
def citys = ["无锡"]
for(def city : citys){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city","region","name","estate_type2"])
def rets = getHouseCases(districts)
ExcelUtils.write("D:\\数据\\江苏\\房源样本.xlsx", rets, city)
}
}
def getHouseCases(districts){
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
rets << ["城市","区域","名称", "物业类型", "最小案例","最大案例", "案例"]
for(def i=0;i<districts.size();i++){
final def d = districts[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def propertyTypeCode = null
if(d.estate_type2=="普通住宅"){
propertyTypeCode = "01"
}else if(d.estate_type2=="别墅"){
propertyTypeCode = "02"
}else if(d.estate_type2=="商住"){
propertyTypeCode = "03"
}
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCase(d.city, d.region, d.name, propertyTypeCode)
if(results){
def row = []
row << d.city
row << d.region
row << d.name
row << d.estate_type2
row << results.minUnitPrice
row << results.maxUnitPrice
def datas = results.datas
if(datas){
def str = ""
for(def r : datas){
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
}
row << str
}
rets << row
}
println totalCount.get()+" "+d.city +" "+d.region+" "+d.name
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def arroundFacilities(){
def districts = CommonDatautils.findAllDistricts("上海", "district", ["city","region","name","estate_type2"])
// districts = districts[0..1000]
def rets = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
rets << ["城市","区域","名称", "重点学校","学校", "公交","地铁","超市","医院","银行"]
for(def i=0;i<districts.size();i++){
final def d = districts[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
totalCount.addAndGet(1)
def row = []
row << d.city
row << d.region
row << d.name
def facilityObj = OpenApiClient.facilitys(d.city, d.region, d.name)
if(facilityObj && facilityObj.result){
def result = facilityObj.result
if(result.primarySchool){
row <<result.primarySchool.join(",").replace('"',"")
}else {
row << ""
}
if(result.school){
row << result.school.join(",").replace('"',"")
}else {
row << ""
}
if(result.bus){
row << result.bus.join(",").replace('"',"")
}else{
row << ""
}
if(result.metro){
row << result.metro.join(",").replace('"',"")
}else{
row << ""
}
if(result.supermarket){
row << result.supermarket.join(",").replace('"',"")
}else{
row << ""
}
if(result.hospital){
row << result.hospital.join(",").replace('"',"")
}else{
row << ""
}
if(result.bank){
row << result.bank.join(",").replace('"',"")
}else{
row << ""
}
}
println totalCount.get() + " "+d.region +" "+d.name
rets << row
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
ExcelUtils.write("D:\\数据\\江苏\\南京周边配套.xlsx", rets, "南京")
}
def arroundFacilitiesByAddress(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B']){
def facilityObj = OpenApiClient.facilitysByAddress(d['A']+d['B'])
if(facilityObj && facilityObj.result){
def result = facilityObj.result
d['G'] = result.city
d['H'] = result.region
d['I'] = result.districtName
if(result.primarySchool){
d['J'] = result.primarySchool.join(",").replace('"',"")
}else {
d['J'] = ""
}
if(result.school){
d['K'] = result.school.join(",").replace('"',"")
}else {
d['K'] = ""
}
if(result.bus){
d['L'] = result.bus.join(",").replace('"',"")
}else{
d['L'] = ""
}
if(result.metro){
d['M'] = result.metro.join(",").replace('"',"")
}else{
d['M'] = ""
}
if(result.bank){
d['N'] = result.bank.join(",").replace('"',"")
}else{
d['N'] = ""
}
if(result.supermarket){
d['O'] = result.supermarket.join(",").replace('"',"")
}else{
d['O'] = ""
}
if(result.hospital){
d['P'] = result.hospital.join(",").replace('"',"")
}else{
d['P'] = ""
}
}
}
})
}
def jsAbcHouseCastTestByAddress(path){
ExcelUtils.multiThreadingModify(path,{d->
if(d['B'] && !d['H']){
def results = com.fangjia.script.gj.OpenApiUtils.jsHouseCaseByAddress(d['A']+d['B'], d['C'])
if(results){
d['H'] = results.minUnitPrice
d['I'] = results.maxUnitPrice
def datas = results.datas
if(datas){
def str = ""
for(def r : datas){
str = str + r.buildingName+"-"+r.strctureArea+"-"+r.unitPrice+"-"+r.sampleDate.substring(0,10)+"\n"
}
d['J'] = str
}
}
}
});
}
def houseValueInformation(path){
ExcelUtils.multiThreadingModify(path,{d->
def results = com.fangjia.script.gj.OpenApiUtils.houseValueInformation(d['A'], d['B'], d['C'])
if(results) {
def addressValue = results.addressValue
if (addressValue) {
d['D'] = addressValue.city + "-" + addressValue.district + "-" + addressValue.name + "-" + addressValue.avgPrice + "-" + addressValue.totalPrice
}
def communityAvgPrice = results.communityAvgPrice
if (communityAvgPrice) {
d['E'] = communityAvgPrice.toString()
}
def surroundingPoi = results.surroundingPoi
if (surroundingPoi) {
d['F'] = surroundingPoi.toString()
}
def house = results.houseCase
if (house) {
d['G'] = house.minUnitPrice
d['H'] = house.maxUnitPrice
def datas = house.datas
if (datas) {
def str = ""
for (def r : datas) {
str = str + r.buildingName + "-" + r.strctureArea + "-" + r.unitPrice + "-" + r.sampleDate.substring(0, 10) + "\n"
}
d['J'] = str
}
}
}
})
}
}
@@ -39,8 +39,10 @@ class DistrictFieldAddBatch extends BaseBatch{
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//
def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
//"alias":"alias",
//ownership_type:权属类型 为商品房、经适房、动迁房等
@@ -51,6 +53,9 @@ class DistrictFieldAddBatch extends BaseBatch{
"build_type":"build_type", "land_property":"ownership_type","live_date":"live_date",
"loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段
// def fieldsMap = ["household_count":"households"]
def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商",
"virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型",
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
@@ -67,7 +72,7 @@ class DistrictFieldAddBatch extends BaseBatch{
@Override
void process(Object args) {
def citys = CommonDatautils.findAllCity().reverse()
def citys = CommonDatautils.findAllCity()
def analysRets = []
@@ -88,12 +93,12 @@ class DistrictFieldAddBatch extends BaseBatch{
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
if(totalCount){
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)
analysRets = addFieldFromMatch(c, totalCount, analysRets)
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
}
}
println analysRets.toString()
sendStatEmail(headers, analysRets, rh_analysRets)
// sendStatEmail(headers, analysRets, rh_analysRets)
// ExcelUtils.write("D:\\数据\\小区\\覆盖率.xlsx", analysRets, "数据")
}
@@ -106,11 +111,11 @@ class DistrictFieldAddBatch extends BaseBatch{
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
def updateRets = processFieldsFromMatch(c , matchRets2)
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%"
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
//updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def pre1 = 0
def pre2 = 0
@@ -199,11 +204,13 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def processFieldsFromMatch(c,matchRets){
def processFieldsFromMatch(c,matchRets, esWebSource){
def fields = fieldsMap.keySet() as List
fields << "city"
fields << "estate_type2"
def fieldValueMap = [:]
def rets = []
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
for(def matchDist : matchRets){
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
@@ -212,11 +219,13 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newValues = strandField(c, dist, matchDist)
if(newValues){
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
}
}
// ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource)
return fieldValueMap;
}
@@ -332,7 +341,7 @@ class DistrictFieldAddBatch extends BaseBatch{
def findEsDatas(c, esWebSource){
def list = []
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r->
ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' ", {r->
list << r
})
return list
@@ -379,6 +388,7 @@ class DistrictFieldAddBatch extends BaseBatch{
try{
totalCount.addAndGet(1)
def ret = matching(data)
// def ret = newMatching(data)
if(ret){
// println totalCount.get() +" "+data.city+" "+data.name+" [Match] "+data.match_region+" "+data.match_name +" "+data.flag+" "+data.credit
rets << ret
@@ -399,7 +409,7 @@ class DistrictFieldAddBatch extends BaseBatch{
def selectRestByCredit(matchRets){
def matchRets1 = []
for(def r : matchRets){
if(r.credit && r.credit>=0.9){
if(r.credit && r.credit>0.9){
matchRets1 << r
}
}
@@ -458,6 +468,58 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newMatching(data){
def name = data.name
if(data.name && data.name.contains("·")){
name = data.name.replace("·", "")
}
def flag = "名称匹配"
def dist = matchDist(data.city, data.region, name)
if(!dist){
dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district")
flag = "映射别名匹配"
}
if(!dist) {
def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district")
if(rets && rets.size()==1){
flag = "别名匹配"
dist = rets[0]
}
}
if(dist){
data.flag = flag
data.match_id = dist._id.toString()
data.match_region = dist.region
data.match_name = dist.name
data.credit = 0.95
}else{
def jsonObject = FangjiaClientUtils.newMatch(data.city, data.region+name, "property", false)
if(jsonObject!=null && jsonObject.credit>0){
if(data.address && isStandardRoad(data.address)){
jsonObject = FangjiaClientUtils.newMatch(data.city, data.address, "property", false)
}else{
data.flag = "匹配接口-地址"
}
}else{
data.flag = "匹配接口-名称"
}
if(jsonObject != null && jsonObject.credit>0) {
data.match_id = jsonObject.get("id")
data.match_region = jsonObject.get("district")
data.match_name = jsonObject.get("name")
data.credit = jsonObject.get("credit")
}
}
if(data.match_id){
return data
}
return null
}
def updateDB(c, updateRets){
for(def rm : updateRets) {
@@ -114,6 +114,27 @@ class FangjiaClientUtils {
return jsonObj;
}
public static def structure(def city, def address){
def jsonObj = null
def content = null
try {
if(address){
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
}
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def arroundPrice(city,address){
def jsonObj = null
def content = null
@@ -172,4 +193,179 @@ class FangjiaClientUtils {
}
}
public static def addrEvaluation(city,region,address, size, floor, maxFloor, room, hall){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://localhost:8080/fangjia-open/property/addressEvaluate?city="+URLEncoder.encode(city,"UTF-8")+"&district="+URLEncoder.encode(region,"UTF-8")+"&address="+URLEncoder.encode(address,"UTF-8")+"&size=${size}&sampleCode=1"
if(floor){
url = url + "&floor="+floor.toString().toDouble().toInteger()
}
if(maxFloor){
url = url + "&maxFloor="+maxFloor.toString().toDouble().toInteger()
}
if(room){
url = url + "&room="+room.toString().toDouble().toInteger()
}
if(hall){
url = url + "&hall="+hall.toString().toDouble().toInteger()
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def newMatch(city,address, property, useMap){
def jsonObj = null
def content = null
def url = null;
try {
address = address.replace("{","").replace("\\","")
url = "http://192.168.0.100:32080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
if(property){
url = url + "&category=${property}"
}
// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println "±¨´íµØÖ·£º"+url
e.printStackTrace();
}
return jsonObj;
}
public static def newMatchPrice(city,address, property, useMap){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://open.fangjia.com/addrSvc/avgPrice?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
if(property){
url = url + "&category=${property}"
}
url = url + "&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def newStructureByLevel9(city,address){
def jsonObj = null
def content = null
try {
address = address.replace("{","").replace("\\","")
def url = "http://192.168.0.100:32080/FangJiaAddressService/v2/structure?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def newStandard(address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/format?address="+URLEncoder.encode(address,"UTF-8")
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def formatFivelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def formatNinelevel(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/structureAndComplete?address="+URLEncoder.encode(address,"UTF-8")+"&complete=true&addressAssessment=true"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
public static def villageIdentify(city, address){
def jsonObj = null
def content = null
try {
address = address.replace("{","")
def url = "http://192.168.0.19:31080/FangJiaAddressService/villageIdentify?address="+URLEncoder.encode(address,"UTF-8")+"&type=2"
if(city){
url = url +"&city="+URLEncoder.encode(city,"UTF-8")
}
content = new HttpRequest().newGet(url, null, "utf-8")
jsonObj = JSONObject.fromObject(content)
println jsonObj
} catch (Exception e) {
println address +" "+content
e.printStackTrace();
}
return jsonObj;
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.district
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -69,14 +70,15 @@ class OfficeFieldCleanBatch extends BaseBatch{
// strandField(offices)
// }
// addBlock()
// addFieldValue()
// updateErrorField()
export()
test()
// export()
// test()
// addOffice()
fileAddBlock()
}
def export(){
@@ -706,7 +708,34 @@ class OfficeFieldCleanBatch extends BaseBatch{
})
}
def fileAddBlock(){
def rets = findAllBlockRange()
ExcelUtils.multiThreadingModify("D:\\数据\\其它\\上海商铺房源.xlsx",{d->
def location = d['T']
if(location && !d['F']){
def arrs = location.split(",")
// 高德转百度
def baidu = BaiDuApiUtils.geoconv(arrs[0].toString().toDouble(), arrs[1].toString().toDouble(), 3)
if(baidu){
def lng2 = baidu.x
def lat2 = baidu.y
def lastObj = null
for (def r : rets) {
def flag = InsidePolygon.isInArea(lng2, lat2, r.points)
if (flag && r.name) {
lastObj = r;
break;
}
}
if (lastObj) {
d['F'] = lastObj.name
}
}
}
})
}
}
@@ -0,0 +1,42 @@
package com.fangjia.script.gj.district
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def city = "常州"
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"])
println districts.size()
def rets = []
rets << ["城市", "区域", "名称", "房价网ToC", "房价网ToB", "近三个月成交均价"]
def index = 0
for(def d : districts){
index++
def ret = []
def dbo = CommonDatautils.queryDistrictPrice(d.city, d.region, d.name, DateUtils.parseFromTightByPattern("2023-09-17","yyyy-MM-dd"))
ret << d.city
ret << d.region
ret << d.name
if(dbo){
ret << dbo.avg_price
ret << dbo.avg_price_2b
}
def map = CcbCommonUtils.findTrades(d, "2023-05-01","2023-09-01")
if(map){
ret << map.avg_price
}
rets << ret
println index
}
ExcelUtils.write("d:\\数据\\江南农商行\\常州小区价格.xlsx", rets, "数据")
@@ -0,0 +1,76 @@
package com.fangjia.script.gj.districtwash
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
class DistrictTypeUpdate {
static main(args) {
def task = new DistrictTypeUpdate();
// task.run()
task.updateType()
}
def updateType(){
ExcelUtils.modify("D:\\数据\\厦门\\物业类型可疑小区(第二批).xlsx",{d->
def flag = d['F']
if(flag && flag.toString().toDouble().toInteger()==2){
def name = d['C']
def type = null
if(name && name.toString().contains("商住")){
type = "商住"
}else{
type = "别墅"
}
def udata = [:]
udata.put("status", "0")
udata.put("cat", "district")
udata.put("username", "高娟")
udata.put("city", d['A'])
udata.put("region", d['B'])
udata.put("name", d['C'])
udata.put("estateType2", type)
def s = FangjiaClientUtils.saveOrUpdateDistrict(udata)
println d['C'] +" "+s
}
})
}
def run(){
def citys = CommonDatautils.findAllCity()
def datas = []
citys.each{c->
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name","estate_type2", "c_date"])
districts.each{d->
def row = null
if(d.name.contains("商住")){
if(d.estate_type2!="商住") {
row = [d.city, d.region, d.name, d.estate_type2, DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd")]
}
}else{
if(d.name.contains("墅")){
if(d.estate_type2!="别墅"){
row = [d.city, d.region, d.name, d.estate_type2, DateUtils.formatDateByPattern(d.c_date, "yyyy-MM-dd")]
}
}
}
if(row){
datas << row
println row
}
}
}
ExcelUtils.write("d:\\数据\\小区\\物业类型可疑小区.xlsx", datas, "小区")
}
}

Some files were not shown because too many files have changed in this diff Show More