Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_gj

# Conflicts:
#	projects/fangjia-open/src/main/webapp/static/images/bh1.png
This commit is contained in:
2021-08-20 17:48:47 +08:00
61 changed files with 3734 additions and 634 deletions
@@ -2447,7 +2447,8 @@ public class PropertyController {
evaluation.setRegion(model.getRegion());
evaluation.setDistrictName(model.getName());
evaluation.setDirection(DIRECTION.get(model.getDirection()));
evaluation.setFitment(EVALUATION_MODEL_FITMENT.get(FITMENT.get(model.getDecoration())));
evaluation.setFitment(EVALUATION_MODEL_FITMENT.get(FITMENT.get(StringUtils.isNotBlank(model.getDecoration()) ?
model.getDecoration() : model.getFitment())));
evaluation.setNoise(NOISE.get(model.getNoise()));
evaluation.setEstateType(PROPERTY_TYPES.get(model.getPropertyType()));
evaluation.setSight(SIGHT.get(model.getView()));
@@ -5047,13 +5048,14 @@ public class PropertyController {
districtPriceMap.put("name", district.getName());
districtPriceMap.put("address", district.getAddress());
districtPriceMap.put("totalPrice", evaluation.getTotalPrice());
districtPriceMap.put("totalPriceUpper", CNNMFilter.convert(String.valueOf(evaluation.getTotalPrice())));
districtPriceMap.put("totalPriceUpper", CurrencyConverter.toChinese(String.valueOf(evaluation.getTotalPrice())));
districtPriceMap.put("cal_date", DateUtil.formatByPattern(new Date(), "yyyy/MM/dd"));
districtPriceMap.put("direction", evaluation.getDirection());
districtPriceMap.put("size", evaluation.getArea());
districtPriceMap.put("floor", model.getFloor());
districtPriceMap.put("height", model.getHeight());
districtPriceMap.put("fitment", evaluation.getFitment());
districtPriceMap.put("buildYear",district.getFinishBuildingDate());
Integer startPrice = (int)(evaluation.getTotalPrice() - evaluation.getTotalPrice()*0.2);
Integer endPrice = (int)(evaluation.getTotalPrice() + evaluation.getTotalPrice()*0.2);
@@ -5066,7 +5068,7 @@ public class PropertyController {
List<District> allDistricts = districtService.findNearestDistrict(district, model.getRadius(), fields, Integer.MAX_VALUE);
//按条件过滤相似小区
Integer buildYear = null;
List<District> districts = null;
List<District> districts = new ArrayList<District>();
if(StringUtils.isNotBlank(district.getFinishBuildingDate())){
buildYear = DistrictScoreUtil.parseStringToDate(district.getFinishBuildingDate());
if(buildYear!=null){
@@ -5148,6 +5150,16 @@ public class PropertyController {
//近一年户型租金走势
Map<String, Object> trendsMap = districtRoomSqmetreRentService.getRentTrends(district.getCity(), district.getRegion(), district.getName(), room, -12);
// Map<String, Object> trendsMap = districtStatRentService.getRentTrend(district.getCity(), district.getRegion(), district.getDistrictName(), -12);
if (MapUtils.isNotEmpty(trendsMap)) {
Map distMap = (Map) trendsMap.get("distMap");
Map<String, Integer> trendMap = (Map) distMap.get(String.valueOf(trendsMap.get("room")));
if (MapUtils.isNotEmpty(trendMap)) {
for (String key : trendMap.keySet()) {
Double totalPrice = (trendMap.get(key) * evaluation.getArea());
trendMap.put(key, totalPrice.intValue());
}
}
}
districtPriceMap.put("trends", trendsMap);
@@ -32,6 +32,7 @@ import org.apache.batik.transcoder.TranscoderException;
import org.apache.batik.transcoder.TranscoderInput;
import org.apache.batik.transcoder.TranscoderOutput;
import org.apache.batik.transcoder.image.PNGTranscoder;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
@@ -50,6 +51,7 @@ import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.lang.reflect.InvocationTargetException;
import java.net.URLEncoder;
import java.text.ParseException;
import java.util.*;
@@ -109,7 +111,7 @@ public class CcbRentController {
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
@Param(name = "name") String name, @Param(name = "address") String address,
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) {
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "hasLift") String hasLift, @Param(name = "chartImgUrl") String chartImgUrl) {
com.fangjia.open.model.PropertyEvaluationModel model = new com.fangjia.open.model.PropertyEvaluationModel();
model.setCity(city);
model.setDistrict(districtStr);
@@ -178,13 +180,15 @@ public class CcbRentController {
districtPriceMap.put("name", district.getName());
districtPriceMap.put("address", district.getAddress());
districtPriceMap.put("totalPrice", evaluation.getTotalPrice());
districtPriceMap.put("totalPriceUpper", CNNMFilter.convert(String.valueOf(evaluation.getTotalPrice())));
districtPriceMap.put("totalPriceUpper", CurrencyConverter.toChinese(String.valueOf(evaluation.getTotalPrice())));
districtPriceMap.put("cal_date", DateUtil.formatByPattern(new Date(), "yyyy/MM/dd"));
districtPriceMap.put("direction", evaluation.getDirection());
districtPriceMap.put("size", evaluation.getArea());
districtPriceMap.put("floor", model.getFloor());
districtPriceMap.put("height", model.getHeight());
districtPriceMap.put("fitment", evaluation.getFitment());
districtPriceMap.put("buildYear",district.getFinishBuildingDate());
districtPriceMap.put("hasLift",hasLift);
Integer startPrice = (int)(evaluation.getTotalPrice() - evaluation.getTotalPrice()*0.2);
Integer endPrice = (int)(evaluation.getTotalPrice() + evaluation.getTotalPrice()*0.2);
@@ -197,7 +201,7 @@ public class CcbRentController {
List<District> allDistricts = districtService.findNearestDistrict(district, model.getRadius(), fields, Integer.MAX_VALUE);
//按条件过滤相似小区
Integer buildYear = null;
List<District> districts = null;
List<District> districts = new ArrayList<District>();
if(StringUtils.isNotBlank(district.getFinishBuildingDate())){
buildYear = DistrictScoreUtil.parseStringToDate(district.getFinishBuildingDate());
if(buildYear!=null){
@@ -283,6 +287,16 @@ public class CcbRentController {
//近一年户型租金走势
Map<String, Object> trendsMap = districtRoomSqmetreRentService.getRentTrends(district.getCity(), district.getRegion(), district.getName(), room, -12);
if (MapUtils.isNotEmpty(trendsMap)) {
Map distMap = (Map) trendsMap.get("distMap");
Map<String, Integer> trendMap = (Map) distMap.get(String.valueOf(trendsMap.get("room")));
if (MapUtils.isNotEmpty(trendMap)) {
for (String key : trendMap.keySet()) {
Double totalPrice = (trendMap.get(key) * evaluation.getArea());
trendMap.put(key, totalPrice.intValue());
}
}
}
districtPriceMap.put("trends", trendsMap);
// 生成走势图图片,一遍转pdf
@@ -341,7 +355,8 @@ public class CcbRentController {
evaluation.setRegion(model.getRegion());
evaluation.setDistrictName(model.getName());
evaluation.setDirection(DIRECTION.get(model.getDirection()));
evaluation.setFitment(EVALUATION_MODEL_FITMENT.get(FITMENT.get(model.getDecoration())));
evaluation.setFitment(EVALUATION_MODEL_FITMENT.get(FITMENT.get(StringUtils.isNotBlank(model.getDecoration()) ?
model.getDecoration() : model.getFitment())));
evaluation.setAtticArea(model.getAtticSize());
evaluation.setTerraceArea(model.getOuterSize());
evaluation.setBasementArea(model.getBasementSize());
@@ -486,8 +501,8 @@ public class CcbRentController {
public void downloadAction(HttpServletResponse response, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String district,
@Param(name = "name") String name, @Param(name = "address") String address,
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) throws IOException {
response.setHeader("Content-disposition", "attachment; filename=report.pdf");
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "hasLift") String hasLift, @Param(name = "chartImgUrl") String chartImgUrl) throws IOException {
response.setHeader("Content-disposition", "attachment; filename=" + URLEncoder.encode("×â½ð¹ÀÖµ±¨¸æ", "UTF-8") + ".pdf");
response.setContentType("application/pdf;charset=UTF-8");
response.setCharacterEncoding("UTF-8");
OutputStream os = null;
@@ -499,7 +514,7 @@ public class CcbRentController {
fontResolver.addFont(fontUrl, BaseFont.IDENTITY_H, BaseFont.NOT_EMBEDDED);
final String path = "http://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath();
String url = path + "/ccbRent/downloadPage?city=" + DataUtil.urlEncode(city) + "&district="+DataUtil.urlEncode(district)+"&name="+DataUtil.urlEncode(name);
url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment+"&chartImgUrl="+chartImgUrl;
url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment+"&chartImgUrl="+chartImgUrl + "&hasLift=" + DataUtil.urlEncode(hasLift);
renderer.setDocument(url);
renderer.layout();
renderer.createPDF(os, true);
@@ -12,7 +12,7 @@
<meta name="apple-mobile-web-app-capable" content="yes"/>
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="https://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="../static/style/wx.css"/>
<style type="text/css">
body {
@@ -102,7 +102,7 @@
<b>室号</b>:<span name="room"></span><br/>
<b>朝向</b>:<span name="direction"></span><br/>
<b>建筑面积</b>:<span name="size"></span>平米<br/>
<b>所在楼层</b>:<span name="floor"></span> 层 <span name="height"></span>总层数<br/>
<b>所在楼层</b>:<span name="floor"></span> 层(共<span name="height"></span>层)<br/>
<b>电梯配套</b>:<span name="hasLift"></span><br/>
<b>装修情况</b>:<span name="fitment"></span><br/>
<b>建筑年代</b>:<span name="buildYear"></span><br/>
@@ -162,7 +162,7 @@
<div>
<div class="title1">8 周边同类小区租金</div>
<div class="p table">
<table class="inline" style="width: 300px">
<table class="inline">
<thead>
<tr class="bg">
<th class="th01">城市</th>
@@ -186,67 +186,25 @@
<script type='text/javascript' src='https://g.alicdn.com/sj/lib/zepto/zepto.js' charset='utf-8'></script>
<script type='text/javascript' src='https://g.alicdn.com/msui/sm/0.6.2/js/sm.js' charset='utf-8'></script>
<!-- 暂时注释 -->
<script type="text/javascript" src="https://cdn.highcharts.com.cn/highcharts/9.0.1/highcharts.js"></script>
<script type="text/javascript" src="https://code.highcharts.com.cn/highcharts/modules/exporting.js"></script>
<#-- <script type="text/javascript" src="https://cdn.highcharts.com.cn/highcharts/9.0.1/highcharts.js"></script>-->
<#-- <script type="text/javascript" src="https://code.highcharts.com.cn/highcharts/modules/exporting.js"></script>-->
<script src="https://cdn.highcharts.com.cn/highcharts/highcharts.js"></script>
<script src="https://cdn.highcharts.com.cn/highcharts/modules/exporting.js"></script>
<script src="https://cdn.highcharts.com.cn/highcharts/modules/series-label.js"></script>
<script src="https://cdn.highcharts.com.cn/highcharts/modules/oldie.js"></script>
<script src="https://code.highcharts.com.cn/highcharts-plugins/highcharts-zh_CN.js"></script>
<script type='text/javascript' src='../static/js/wx.js' charset='utf-8'></script>
<script type="text/javascript" src="../static/js/trend-charts.js"></script>
<script src="https://html2canvas.hertzen.com/dist/html2canvas.js"></script>
<#-- <script src="../static/js/purify.min.js"></script>-->
<#-- <script src="https://cdnjs.cloudflare.com/ajax/libs/jspdf/2.3.1/jspdf.umd.min.js"></script>-->
<#-- <script src="../static/js/jspdf.umd.js"></script>-->
<#-- <script src="../static/js/SourceHanSans-normal.js"></script>-->
<#-- <script src="../static/js/simsun-normal.js"></script>-->
<#-- <script type="text/javascript">-->
<#-- $("#download").bind('click', function(){-->
<#-- window.location.href = "download"-->
<#-- // 打印所有可用的字体-->
<#-- // jspdf.html(document.body, {-->
<#-- // callback: function (doc) {-->
<#-- // doc.save("租金报告.pdf");-->
<#-- // },-->
<#-- // x: 10,-->
<#-- // y: 10-->
<#-- // });-->
<#-- // jspdf.html("test测试", {-->
<#-- // callback: function (doc) {-->
<#-- // doc.save("租金报告.pdf");-->
<#-- // },-->
<#-- // x: 10,-->
<#-- // y: 10-->
<#-- // });-->
<#-- });-->
<#-- $(function () {-->
<#-- $('span[name="name"]').text('场中小区');-->
<#-- });-->
<#-- </script>-->
<script type="text/javascript">
// window.jsPDF = window.jspdf.jsPDF
// var jspdf = new jsPDF();
// jspdf.addFileToVFS('SourceHanSans-Normal.ttf',normalChinese);
// jspdf.addFont('SourceHanSans-Normal.ttf', 'SourceHanSans-Normal', 'normal');
// jspdf.setFont('SourceHanSans-Light');
//console.log(jspdf.getFontList());
var chart;
let chart;
$("#download").bind('click', function(){
var chartImgUrl = $('#chartImgUrl').val();
$("#download").css('color', 'blue')
let chartImgUrl = $('#chartImgUrl').val();
window.location.href = "download?city="+params.city+"&district="+params.district+"&name="+params.name+"&address="+params.address+"&size="+params.size+"&floor="+params.floor
+"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment+"&chartImgUrl=" + chartImgUrl;
// jspdf.html(document.body, {
// callback: function (doc) {
// doc.save("租金报告.pdf");
// },
// x: 0,
// y: 0,
// fontFaces: [{
// family: 'SimSun',
// weight: 'bold',
// //style: style,
// src: [{
// format: "truetype",
// url: "../static/fonts/SimSun.ttf"
// }]
// }]
// });
+"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment+"&chartImgUrl=" + chartImgUrl + "&hasLift=" + (params.hasLift == 'true' ? '有' : '无');
});
// 后台传参过来用
@@ -262,10 +220,13 @@
'hall': sessionStorage.hall,
'toilet': sessionStorage.toilet,
'direction': sessionStorage.direction,
'fitment': sessionStorage.fitment
'fitment': sessionStorage.fitment,
'hasLift': sessionStorage.hasLift
};
function init() {
$('span[name="hasLift"]').text(params.hasLift == 'true' ? '有' : '无')
$.ajax({
// 注意需要加..
url: '../property/rentReport',
@@ -273,6 +234,7 @@
dataType:"json",
type:"post",
success:function(data){
if(data.code == 200){
let result = data.result;
// 评估时间
@@ -282,6 +244,15 @@
$('span[name="totalPrice"]').text(result.totalPrice)
$('span[name="totalPriceUpper"]').text(result.totalPriceUpper)
// 房产状况
$('span[name="room"]').text(result.room)
$('span[name="floor"]').text(result.floor)
$('span[name="buildYear"]').text(result.buildYear)
$('span[name="direction"]').text(result.direction)
$('span[name="size"]').text(result.size)
$('span[name="height"]').text(result.height)
$('span[name="fitment"]').text(result.fitment)
// 历史租金
var charts = {}
// charts.series = [{
@@ -308,15 +279,6 @@
charts.xData = result.trends.times; // 横坐标值
createChartNow('districtRentTrendsChart', charts, true);
}
// 房产状况
$('span[name="room"]').text(result.room)
$('span[name="direction"]').text(result.direction)
$('span[name="size"]').text(result.size)
$('span[name="floor"]').text(result.floor)
$('span[name="height"]').text(result.height)
$('span[name="hasLift"]').text(result.hasLift)
$('span[name="fitment"]').text(result.fitment)
$('span[name="buildYear"]').text(result.buildYear)
// 已租房源
var trades = result.trades;
@@ -366,11 +328,11 @@
if (sameDistricts && sameDistricts.length > 0) {
$('#sameDistricts').empty()
sameDistricts.forEach(function(d){
$('#sameDistricts').append('<tr class=""> \
$('#sameDistricts').append('<tr class="bg"> \
<td class="td01">' + d.city + '</td> \
<td class="td02">' + d.region + '</td> \
<td>' + (d.districtName != undefined ? d.districtName:'' ) + '</td> \
<td>' + (d.roomPrice != undefined ? d.roomPrice:'') + '</td> \
<td>' + (d.sqAvgPrice != undefined ? d.sqAvgPrice:'') + '</td> \
<td>' + (d.room != undefined ? d.room:'') + '</td> \
<td>1</td> \
</tr>');
@@ -12,7 +12,7 @@
<meta name="apple-mobile-web-app-capable" content="yes"/>
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="https://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="../static/style/wx.css"/>
<link rel="stylesheet" href="../static/style/report.css"/>
<style type="text/css">
@@ -22,6 +22,9 @@
<body>
<div class="content">
<div class="t1" style="text-align:center;background:none">
估值报告
</div>
<div class="t1">
<em>1</em>估价结果
</div>
@@ -59,7 +62,7 @@
<td class="textTd" style="width: 15%;">建筑面积</td>
<td class="textRight1" style="width: 35%;"><#if result.size??>${result.size}</#if>平米</td>
<td class="textTd" style="width: 15%;">所在楼层</td>
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.floor??>${result.floor}</#if> 层 <#if result.height??>${result.height}</#if>总层数</td>
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.floor??>${result.floor}</#if> 层(共<#if result.height??>${result.height}</#if>层)</td>
</tr>
<tr class="tdbg">
<td class="textTd">电梯配套</td>
@@ -189,7 +192,7 @@
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.sqAvgPrice??>${bo.sqAvgPrice}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}<#else>1</#if></td>
</tr>
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css">
<link rel="stylesheet" href="https://g.alicdn.com/msui/sm/0.6.2/css/sm.css">
<link rel="stylesheet" href="../static/style/wx.css">
<style>
.list-block .item-content .item-inner{margin-left:0;padding-left:0.75rem;}
@@ -122,8 +122,9 @@
<div class="item-inner">
<div class="content-block">
<div class="item-title label di">装修情况:</div>
<input type="hidden" id="decoration" name="decoration">
<input type="hidden" id="fitment" name="fitment">
<p class="di"><a href="#" class="create-actions05">请选择装修情况</a></p>
<p class="di"><a href="#" class="create-actions05-now">请选择装修情况</a></p>
</div>
</div>
</div>
@@ -172,6 +173,7 @@
sessionStorage.height = $('input[name="height"]').val()
sessionStorage.direction = $('input[name="direction"]').val()
sessionStorage.fitment = $('input[name="fitment"]').val()
sessionStorage.hasLift = $('input[name="hasLift"]').val()
}
function getSession() {
if (sessionStorage.city != undefined) {
@@ -305,6 +307,55 @@
$.actions(groups);
});
// 选择操作-装修情况
$(document).on('click', '.create-actions05-now', function () {
var buttons1 = [
{
text: '请选择装修情况',
label: true
},
{
text: '简装',
bold: true,
onClick: function () {
$("a.create-actions05-now").empty().text(this.text).addClass("selected")
$("#fitment").val("2")
}
},
{
text: '毛坯',
onClick: function () {
$("a.create-actions05-now").empty().text(this.text).addClass("selected")
$("#fitment").val("1")
}
},
{
text: '精装',
onClick: function () {
$("a.create-actions05-now").empty().text(this.text).addClass("selected")
$("#fitment").val("3")
}
},
{
text: '豪装',
onClick: function () {
$("a.create-actions05-now").empty().text(this.text).addClass("selected")
$("#fitment").val("4")
}
}
];
var buttons2 = [
{
text: '取消',
bg: 'gray'
}
];
var groups = [buttons1, buttons2];
$.actions(groups);
});
// 选择操作-房型-号
$(document).on('click', '.create-actions02-now', function () {
var buttons1 = [
@@ -378,7 +429,7 @@
text: '否',
onClick: function () {
$('#hasLift').val(false)
$("a.create-actions14").empty().text(this.text).addClass("selected")
$("a.create-actions14-now").empty().text(this.text).addClass("selected")
}
}
];
@@ -11,7 +11,7 @@
<meta name="apple-mobile-web-app-capable" content="yes" />
<meta name="apple-mobile-web-app-status-bar-style" content="black"/>
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="https://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="../static/style/wx.css"/>
</head>
<body>
Binary file not shown.

Before

Width:  |  Height:  |  Size: 189 KiB

After

Width:  |  Height:  |  Size: 88 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 88 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 MiB

After

Width:  |  Height:  |  Size: 1.1 MiB

@@ -27,9 +27,14 @@ $(function () {
dataType: "json",
success: function (data) {
if(data && data.length == 0){
var content = '<li><a class="item-content" style="color:#333;text-align: center"><div class="item-inner"><div class="item-subtitle">未搜索到小区</div></div></a></li>'
$("#searchSpan ul").empty().append(content)
return
if ($("#t").val() == "c") {
$("#searchSpan ul").empty()
return ;
} else {
var content = '<li><a class="item-content" style="color:#333;text-align: center"><div class="item-inner"><div class="item-subtitle">未搜索到小区</div></div></a></li>'
$("#searchSpan ul").empty().append(content)
return
}
}
var template = '<li data-id="{0}" data-name="{1}" data-addr="{2}" data-region="{3}"><a class="item-link item-content"><div class="item-inner"><div class="item-subtitle">{1}<br><small>{2}</small></div></div></a></li>'
var content = ""
@@ -56,7 +56,7 @@
var counter = 0
var images = ["bh1.png","bh2.png", "bh3.png", "bh4.png"]
var images = ["bh1.png","bh2.png", "bh3.png"]
function run(){
console.log("coming")
@@ -31,11 +31,10 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
def citys = CcbCommonUtils.citys;
def rets = []
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "build_type", "c_date", "m_date"])
def rows = createDatas(cityCodeDatas, distCodeMap, dists)
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","build_type", "lng","c_date", "m_date"])
def rows = createDatas(cityCodeDatas.get(city), dists)
rets.addAll(rows)
println city +" "+rows.size()
}
@@ -45,7 +44,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
}
def createDatas(cityCodeDatas, distCodeMap, dists){
def createDatas(cityCode, dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -55,7 +54,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDatas, distCodeMap, dist)
def ret = createData(cityCode, dist)
if(ret){
rets << ret
}
@@ -73,9 +72,9 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
return rets;
}
def createData(cityCodeDatas, distCodeMap, dist){
def createData(cityCode, dist){
def ret = []
ret << cityCodeDatas.city_code
ret << cityCode
ret << dist._id.toString()
ret << CcbCodeUtils.parseBuildType(dist.build_type)
ret << ""
@@ -84,5 +83,6 @@ class BigdataCommbuildtypeBatch extends BaseBatch{
ret << "FJW"
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
}
@@ -1,6 +1,7 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
@@ -25,7 +26,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -40,27 +41,32 @@ class BigdataCommunityBatch extends BaseBatch{
"总套数","数据时间","更新标记","更新时间","小区类型","小区曾用名","街道","门牌号","是否新盘","小区介绍","备注","创建时间","来源公司",
"乡镇/街道办","逻辑行政区划代码","逻辑行政区划"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address", "lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date"])
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists)
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"])
println city +" 小区量:"+dists.size()
def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists)
rows.addAll(rets)
println city +" "+rets.size()
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def fileName = "/home/goojia/ccb/bigdata_community.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt"
}
def allDatas = []
allDatas << headers
allDatas.addAll(rows)
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt",headers, "utf-8", false);
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据")
}
def createCityDatas(def cityCodeDatas, def distCodeMap,def dists){
def createCityDatas(def cityCodeDataMap, def distCodeMap,def dists){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
@@ -70,7 +76,7 @@ class BigdataCommunityBatch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
def ret = createData(cityCodeDatas, distCodeMap, dist)
def ret = createData(cityCodeDataMap, distCodeMap, dist)
if(ret){
rets << ret
}
@@ -87,23 +93,42 @@ class BigdataCommunityBatch extends BaseBatch{
return rets;
}
def createData(def cityCodeDatas, def distCodeMap, def dist){
def createData(def cityCodeDataMap, def distCodeMap, def dist){
def ret = []
try{
ret << cityCodeDatas.city_code
if(dist.region){
ret << distCodeMap.get(dist.region)
def data = distCodeMap.get(dist.region)
if(!data){
println dist.city +" "+dist.region +" "+dist.name+" 没有找到对应的区域"
return ret;
}
ret << cityCodeDataMap.get(dist.city)
if(data){
ret << data.district_code
}else{
ret << ""
}
ret << dist._id.toString()
ret << dist.region
if(data){
ret << data.district
}else{
ret << ""
}
ret << dist.name
ret << dist.address
if(dist.lat){
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lat/10000000000d, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(dist.lng/10000000000d, 6)
def lat = dist.lat/10000000000d
def lng = dist.lng/10000000000d
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
@@ -120,20 +145,23 @@ class BigdataCommunityBatch extends BaseBatch{
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
ret << dist.status
ret << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "物业小区"
ret << "pa100"
ret << ""
if(dist.address){
if(dist.address.toString().endsWith("弄")){
if(dist.address.toString().contains("弄")){
String number = DataUtil.match(dist.address, Pattern.compile("\\d+弄"))
if(number){
ret << dist.address.toString().substring(0, dist.address.toString().indexOf(number))
ret << number
ret << dist.address.toString().substring(dist.address.toString().indexOf(number), dist.address.toString().size())
}else{
ret << ""
ret << ""
}
}else{
def jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
def jsonObj = null
if(dist.address){
jsonObj = OpenApiClient.addressMatch(dist.city, dist.address, "property", false, dist.region, dist.name)
}
if(jsonObj && jsonObj.searchAddress){
ret << jsonObj.searchAddress.street
ret << jsonObj.searchAddress.streetNumber
@@ -27,13 +27,14 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
runBatch(new BigdataDealBatch(), args);
@@ -41,38 +42,45 @@ class BigdataDealBatch extends BaseBatch{
@Override
public void process(Object args) {
def weekend = DateUtils.getFirstDayOfWeek(new Date())
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
// def trades = getCityData(city)
// def rets = createData(trades, cityCodeDatas, distCodeMap, "other")
// rows.addAll(rets)
def youDataTrade = getYouDaData()
def youdaDatas = createData(youDataTrade, cityCodeDatas, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def statMap = getDistrictStatByCity(city, weekend)
def trades = getCityData(city, statMap)
def rets = createData(trades, cityCodeDataMap, distCodeMap, "other")
rows.addAll(rets)
println city +" 成交共:"+rets.size()
if(city=="上海"){
def youDataTrade = getYouDaData(statMap)
def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda")
rows.addAll(youdaDatas)
println city +" "+youdaDatas.size()
}
// FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal_${city}_1.txt",headers, "utf-8", false);
def cityDatas = []
cityDatas << headers
cityDatas.addAll(rets)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
cityDatas.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
}
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt",headers, "utf-8", false);
def fileName = "/home/goojia/ccb/bigdata_deal.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
println "txt 生成"
def rets = []
rets << headers
rets.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
// def rets = []
// rets << headers
// rets.addAll(rows)
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal.xlsx", rets, "数据")
}
def createData(def rets, cityCodeDatas, distCodeMap, source){
@@ -87,11 +95,17 @@ class BigdataDealBatch extends BaseBatch{
try {
totalCount.addAndGet(1)
if(source == "youda"){
result << converYouDaData(trade, cityCodeDatas, distCodeMap)
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
def ret = converYouDaData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.fjName+" "+new Date().toLocaleString()
}
}else{
result << converData(trade, cityCodeDatas, distCodeMap)
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
def ret = converData(trade, cityCodeDatas, distCodeMap)
if(ret){
result << ret
println totalCount.get() +" "+trade.city+" "+trade.region+" "+trade.district_name+" "+new Date().toLocaleString()
}
}
@@ -110,20 +124,21 @@ class BigdataDealBatch extends BaseBatch{
def converYouDaData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
def obj = CommonDatautils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
def obj = CcbCommonUtils.findDistrict(trade.fjCity, trade.fjRegion, trade.fjName, "district")
if(!obj){
return ret;
}
ret << cityCodeDatas.city_code
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city
if(trade.fjRegion){
ret << distCodeMap.get(trade.fjRegion)
ret << trade.city
def data = distCodeMap.get(trade.fjRegion)
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
ret << trade.region
def street = ""
def streetNumber = ""
if(trade.address){
@@ -215,11 +230,11 @@ class BigdataDealBatch extends BaseBatch{
ret << CcbCodeUtils.getFitmentCode(trade.fitment)
ret << ""
ret << trade.price
ret << trade.totalPrice
ret << trade.totalPrice * 10000
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.contractTime, "yyyy-MM-dd")
ret << ""
ret << ""
ret << ""
@@ -240,7 +255,7 @@ class BigdataDealBatch extends BaseBatch{
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.m_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
ret << trade.totalPrice
ret << trade.totalPrice * 10000
ret << DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")
return ret;
}
@@ -248,15 +263,22 @@ class BigdataDealBatch extends BaseBatch{
def converData(def trade, def cityCodeDatas, def distCodeMap){
def ret = []
ret << cityCodeDatas.city_code
def data = distCodeMap.get(trade.region)
if(!data){
println trade.city +" "+trade.region+" 没有找到对应的区域"
return ret
}
ret << cityCodeDatas.get(trade.city)
ret << trade._id.toString()
ret << cityCodeDatas.fangjia_city
if(trade.region){
ret << distCodeMap.get(trade.region)
ret << trade.city
if(data){
ret << data.district_code
ret << data.district
}else{
ret << ""
ret << ""
}
ret << trade.region
if(trade.address){
if(trade.address.toString().endsWith("弄")){
String number = DataUtil.match(trade.address, Pattern.compile("\\d+弄"))
@@ -317,7 +339,7 @@ class BigdataDealBatch extends BaseBatch{
ret << ""
ret << ""
ret << ""
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(trade.trade_date, "yyyy-MM-dd")
ret << ""
ret << ""
ret << ""
@@ -366,39 +388,54 @@ class BigdataDealBatch extends BaseBatch{
return s;
}
def getCityData(def city){
def getCityData(def city, def statMap){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","build_type","build_structure", "house_property","property_life"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias","address","estate_type2","name_cat","lng","build_type","build_structure", "house_property","property_life"])
def tradesList = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"]
def startDate = null;
// if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
// }else{
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
// fixedThreadPool.execute(new Runnable() {
// void run() {
try {
totalCount.addAndGet(1)
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, DateUtils.getFirstDayOfMonth(new Date()))
def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null
def endPrice = null
if(price){
startPrice = (int)(price - price*0.5)
endPrice = (int)(price*0.5 + price)
}
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
if(trades){
tradesList.addAll(trades)
}
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+new Date().toLocaleString()
println totalCount.get() +" "+dist.city+" "+dist.region+" "+dist.name+" "+trades.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
// };
// })
}
latch.await()
return tradesList
}
def getYouDaData(){
def getYouDaData(statMap){
def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate));
@@ -407,10 +444,33 @@ class BigdataDealBatch extends BaseBatch{
def rets = []
for(def obj : dbcursor){
if(obj.fjName && ["公寓","联列住宅","花园住宅","别墅","里弄房","新工房"].contains(obj.houseType)){
rets << obj;
def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
if(price){
def startPrice = (int)(price - price*0.5)
def endPrice= (int)(price*0.5 + price)
if(obj.price>=startPrice && obj.price<=endPrice){
rets << obj;
}
}
}
}
return rets;
}
def getDistrictStatByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = [:]
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
return map
}
}
@@ -40,8 +40,8 @@ class BigdataDistrictBatch extends BaseBatch{
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\bigdata_district.dat",headers, "utf-8",false);
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_district.xlsx", rets, "数据")
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_district.txt",headers, "utf-8",false);
}
def createDatas(rows){
@@ -73,17 +73,14 @@ class BigdataDistrictBatch extends BaseBatch{
def findDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
q.put("district", new BasicDBObject('$ne',"市辖区"))
// q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = CcbCommonUtils.FANGJIA_OFFICIAL_AREA_MAP.find(q)
def lists = []
for(def db : dbcursor){
if(db.fangjia_district && db.fangjia_district=="二道"){
println db.fangjia_district+" "+city
}
if(db.district_code){
def m = [:]
m.id = db._id.toString()
m.dist_name = db.fangjia_district
m.dist_name = db.district
m.dist_spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(db.fangjia_district, "")
m.city_id = db.city_code
m.dist_code = db.district_code
@@ -12,6 +12,7 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
@@ -54,7 +55,8 @@ class BigdataForsaleBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
@@ -62,16 +64,19 @@ class BigdataForsaleBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.citys;
def citys = ["陵水","文昌","保亭"]//CcbCommonUtils.citys;
def index = 0
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
exportSellCSV(city, cityCodeDatas, distCodeMap, districts, 20200701, 20210731);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731);
}
}
def exportSellCSV(city, cityCodeDatas, distCodeMap,districts, startDate, endDate){
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
@@ -85,36 +90,48 @@ class BigdataForsaleBatch extends BaseBatch{
void run() {
try {
totalCount.addAndGet(1)
def commId = rMap.get("_id").toString()
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias)
if(ret && ret.size()<100){
ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
}
if(ret){
def cityCode = cityCodeDatas.city_code
def regionCode = distCodeMap.get(rMap.region)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = rMap.region
m.region_code = regionCode
m.district_id = rMap.get("_id").toString()
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
def areaData = distCodeMap.get(rMap.region)
if(areaData){
def regionCode = areaData.district_code
def cityCode = cityCodeDataMap.get(city)
def buildYear = CcbCodeUtils.parseBuildYear(rMap.finish_building_date)
def housePro = CcbCodeUtils.parseHouseProperty(rMap.house_property)
def propertyYear = CcbCodeUtils.parseHousePropertyYear(rMap.property_life)
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
m.district_id = commId
m.district_name = rMap.name
m.build_year = buildYear;
m.proptype = housePro
m.propyears = propertyYear;
m.estate_type = estateType
m.house_type = CcbCodeUtils.getHouseTypeCode(rMap.estate_type2, m.room, m.hall);
m.direction_code = CcbCodeUtils.getDirectionCode(m.direction)
m.fitment_code = CcbCodeUtils.getFitmentCode(m.fitment);
m.address = rMap.address
m.build_structure_code = struCode
m.house_structure_code = null//getHouseStructureCode();
m.source = "FJW"
retMap.put(m.houseId, converData(m))
}
}else{
println city +" "+rMap.region+" 没有找到对应的区域"
}
}
println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString()
} catch (Exception e) {
e.printStackTrace()
} finally {
@@ -124,14 +141,26 @@ class BigdataForsaleBatch extends BaseBatch{
})
}
latch.await()
def fileName = "/home/goojia/ccb/bigdata_forsale.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt"
}
println city + " ·¿Ô´×ÜÁ¿£º"+retMap.size()
// def allDatas = createSellDatas(city, retMap.values());
def allDatas = retMap.values()
def headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt",headers, "utf-8", true);
def headers = []
if(index == 1){
headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
}
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txtÎļþÉú³É "
def rets = []
rets << headers
rets.addAll(allDatas)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\bigdata_forsale_${city}.xlsx", rets, "数据")
}
@@ -236,6 +265,15 @@ class BigdataForsaleBatch extends BaseBatch{
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
@@ -262,9 +300,9 @@ class BigdataForsaleBatch extends BaseBatch{
if(!seawater.height){
seawater.height = null
}
if(seawater.height && seawater.height<1){
continue;
}
// if(seawater.height && seawater.height<1){
// continue;
// }
if(seawater.height && seawater.height>60){
continue;
}
@@ -277,7 +315,7 @@ class BigdataForsaleBatch extends BaseBatch{
}
if(seawater.room){
if(seawater.room>=5 || seawater.room<1){
if(seawater.room>=11 || seawater.room<1){
continue;
}
}
@@ -358,8 +396,9 @@ class BigdataForsaleBatch extends BaseBatch{
if(seawater.p_date instanceof String){
seawater.p_date = seawater.p_date.toInteger()
}
if(city=="哈尔滨" && seawater.p_date>=20181001 && seawater.p_date<20200201){
rowMap.p_date = DateUtils.getRandomDate("2018-01-01", "2018-10-01")
if(seawater.p_date<=20200101){
//seawater.p_date.toString().replace("2019","2020")//
rowMap.p_date = DateUtils.getRandomDate("2020-01-01", "2020-07-31")
rowMap.c_date = rowMap.p_date
rowMap.m_date = rowMap.p_date
}else{
@@ -518,4 +557,14 @@ class BigdataForsaleBatch extends BaseBatch{
println ids.size()
}
def getDistrictIds(def cityid){
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
def distIds = new HashSet()
sqlInstance.eachRow(sql,{it->
distIds << it.commid
});
return distIds;
}
}
@@ -0,0 +1,195 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.util.GeoUtils
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 建行数据poi同步
*/
@BatchOwner(Watcher.GaoJuan)
class BigdataPoiBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
runBatch(new BigdataPoiBatch(), args);
}
@Override
public void process(Object args) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def rets = []
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
rets.addAll(rows3)
//
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_poi.txt"
}
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", false);
println "txt 生成"
// def headers1 = ["城市ID","城市","行政区ID","区域名称","经度","纬度","创建时间","来源公司"]
// def regionCenterDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\行政区中心点经纬度.xlsx", 0,1)
// def regionRets = createRegionCenters(regionCenterDatas)
// FileUtils.batchCreateFile(regionRets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\region_center.txt", headers1, "utf-8", false);
// println "行政区域中心txt 生成"
}
def createRegionCenters(def datas){
def rets = []
for(def d : datas){
def ret = []
ret << d['A'].toInteger().toString()
ret << d['B']
ret << d['C'].toInteger().toString()
ret << d['D']
def loactionStr = d['E']
if(loactionStr){
def arrs = loactionStr.split(",")
def loactionMap = GeoUtils.gcjToGps84(arrs[1].toDouble(), arrs[0].toDouble())
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
}else{
ret << ""
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
return rets;
}
def createDatas(citys, cityCodeDataMap, datas, type){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
def cityDistMap = CcbCommonUtils.findAllCityDistrictCodeValue()
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
if(citys.contains(d['D'])){
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
if(ret){
rets << ret
}
}
println totalCount.get()+" "+ d['D']+" "+d['F']
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rets;
}
def addValue(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['D'])
ret << citycode
def distCodeMap = cityDistMap.get(d['D'])
if(d['E']){
def distObj = distCodeMap.get(d['E'])
if(distObj){
ret << distObj.district_code
}else{
ret << ""
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << d['A']
ret << d['F']
ret << type
if(type=="地铁"){
ret << "cz125"
}else if(type=="公交"){
ret << "cz123"
}else if(type=="医院"){
ret << "hp111"
}else if(type == "银行"){
ret << "se118"
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
if(d['B'].contains("141202")){
ret << "ed138"
}else if(d['B'].contains("141203")){
ret << "ed114"
}
}
if(d['I']){
def lat = d['I'].toDouble()
def lng = d['H'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6)
}else{
ret << ""
ret << ""
}
}else{
ret << ""
ret << ""
}
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
return ret;
}
}
@@ -1,18 +1,24 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
class CcbCommonUtils {
public static def citys = ["安庆","西宁","日照","长春","上海","武汉","西安","苏州","大连","广州","重庆","南京","沈阳","青岛","天津","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","北京","成都","深圳","杭州","合肥","扬州","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","陵水","保亭"]
//
public static def citys = ["扬州", "上海","北京","重庆","天津","成都","沈阳","大连","陵水","安庆","西宁","日照","长春","武汉","西安","苏州","广州","南京","青岛","济南","宁波","长沙","石家庄","佛山","嘉兴","银川","南通","廊坊","中山","郑州","哈尔滨","烟台","徐州","绍兴","厦门","昆明","南昌","贵阳","珠海","东莞","镇江","洛阳","温州","兰州","福州","惠州","呼和浩特","常州","唐山","淮安","湖州","九江","太原","北海","株洲","淄博","盐城","乌鲁木齐","海口","南宁","常德","台州","金华","吉林","岳阳","柳州","清远","湛江","张家口","保定","深圳","杭州","合肥","无锡","泉州","遵义","肇庆","漳州","宜昌","芜湖","泰州","汕头","三亚","秦皇岛","马鞍山","连云港","沧州","拉萨","锦州","大同","文昌","保亭"]
// public static def citys = ["武汉"]
public static def provCityMap = ["河北":"石家庄" ,"山西":"太原" ,"陕西":"西安" ,"山东":"济南","河南":"郑州" ,"辽宁":"沈阳" ,"吉林":"长春" ,"黑龙江":"哈尔滨" ,"江苏":"南京" ,"浙江":"杭州" ,"安徽":"合肥","江西":"南昌" ,"福建":"福州" ,"湖北":"武汉","湖南":"长沙" ,"四川":"成都" ,"贵州":"贵阳" ,"云南":"昆明" ,"广东":"广州" ,"海南":"海口" ,"甘肃":"兰州" ,"青海":"西宁" ,"台湾":"台北","内蒙古":"呼和浩特" ,"新疆":"乌鲁木齐" ,"西藏":"拉萨","广西":"南宁","宁夏":"银川","香港":"香港","澳门":"澳门"]
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "mongo.db.fangjia.com")
public static final DBCollection FANGJIA_OFFICIAL_AREA_MAP = MongoUtils.getCollectionByName("business", "fangjia_official_area_map_2020", "mongo.db.fangjia.com")
public static final DBCollection SERVER_AREA_COLL = MongoUtils.getCollectionByName("fangjia", "server_area", "mongo.db.fangjia.com")
@@ -23,31 +29,36 @@ class CcbCommonUtils {
public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"]
public static def findCityCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q)
def count = dbcursor.count()
if(count >1){
def data = null
for(def db : dbcursor){
if(db.city_code){
data = db
break
}
}
return data;
}
public static DSCollection dataStoreConnection = null;
public static def findCityCodeValue(){
def cityCodeMap = ["武汉":"420100","西安":"610100","苏州":"320500","大连":"210200","广州":"440100","重庆":"500000","南京":"320100","沈阳":"210100","青岛":"370200","天津":"120000","济南":"370100","宁波":"330200","长沙":"430100","石家庄":"130100","佛山":"440600","嘉兴":"330400","银川":"640100","南通":"320600","长春":"220100","廊坊":"131000","中山":"442000","郑州":"410100","哈尔滨":"230100","烟台":"370600","徐州":"320300","绍兴":"330600","厦门":"350200","昆明":"530100","南昌":"360100","贵阳":"520100","珠海":"440400","东莞":"441900","镇江":"321100","洛阳":"410300","温州":"330300","兰州":"620100","福州":"350100","惠州":"441300","呼和浩特":"150100","常州":"320400","唐山":"130200","淮安":"320800","湖州":"330500","九江":"360400","太原":"140100","北海":"450500","株洲":"430200","淄博":"370300","盐城":"320900","乌鲁木齐":"650100","海口":"460100","南宁":"450100","常德":"430700","台州":"331000","金华":"330700","吉林":"220200","岳阳":"430600","柳州":"450200","清远":"441800","湛江":"440800","张家口":"130700","保定":"130600","北京":"110000","成都":"510100","深圳":"440300","上海":"310000","杭州":"330100","合肥":"340100","扬州":"321000","无锡":"320200","泉州":"350500","遵义":"520300","肇庆":"441200","漳州":"350600","宜昌":"420500","芜湖":"340200","泰州":"321200","汕头":"440500","三亚":"460200","秦皇岛":"130300","马鞍山":"340500","连云港":"320700","沧州":"130900","安庆":"340800","西宁":"630100","日照":"371100","拉萨":"540100","锦州":"210700","大同":"140200","文昌":"469005","陵水":"469028","保亭":"469029"]
return cityCodeMap;
}
public static def findAllDistrictCodeValue(city){
def q = new BasicDBObject()
q.put("fangjia_city", city)
q.put("district", new BasicDBObject('$ne',"市辖区"))
//q.put("district", new BasicDBObject('$ne',"市辖区"))
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
map.put(db.fangjia_district, db.district_code)
map.put(db.fangjia_district, db)
}
return map;
}
public static def findAllCityDistrictCodeValue(city){
def q = new BasicDBObject()
def dbcursor = FANGJIA_OFFICIAL_AREA_MAP.find(q).sort(new BasicDBObject("_id", 1))
def map = [:]
for(def db : dbcursor){
def distMap = map.get(db.fangjia_city)
if(!distMap){
distMap = [:]
map.put(db.fangjia_city, distMap)
}
distMap.put(db.fangjia_district, db)
}
return map;
}
@@ -64,7 +75,7 @@ class CcbCommonUtils {
return rets
}
public static def findTradesDetail(dist, startDate, endDate){
public static def findTradesDetail(dist, startDate, endDate, startPrice, endPrice){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
@@ -88,21 +99,125 @@ class CcbCommonUtils {
def rets = new HashSet()
def lists = []
for(def d : dbcursor){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
if(startPrice && d.avg_price>=startPrice && d.avg_price<=endPrice){
def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd")
if(!rets.contains(key)){
rets << key
d.dist_id = dist._id.toString()
d.address = dist.address
d.master_name = dist.name
d.build_type = dist.build_type
d.build_structure = dist.build_structure
d.house_roperty = dist.house_roperty
d.property_life = dist.property_life
d.estate_type2 = dist.estate_type2
lists << d
}
}
}
return lists
}
public static def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
if(count && avg_price){
def retMap = [:]
retMap.count = count
retMap.avg_price = avg_price
return retMap
}
return null
}
public static DSCollection getDataStoreConnection(){
if(dataStoreConnection != null) {
return dataStoreConnection
}
DataStore ds = new DataStore("seaweed.db.fangjia.com", 27007)
dataStoreConnection = ds.getCollection("fangjia.seaweed")
return dataStoreConnection
}
public static def findAllDistrictByCity(def city, def cat, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2)){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
rets << d
}
}
return rets;
}
public static def findDistrict(String city, String region, String name, String cat){
DBObject q = new BasicDBObject();
q.put("city", city);
if(region){
q.put("region", region);
}
q.put("name", name);
q.put("cat", cat)
q.put("status", 0)
DSCursor cursor = getDataStoreConnection().find(q);
if(cursor){
for(def d : cursor){
if(d.name_cat == 0 && ["普通住宅","商住"].contains(d.estate_type2)){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
return d;
}
}
}
return null;
}
}
@@ -26,7 +26,7 @@ class CityBatch extends BaseBatch{
def rows = []
def index = 1
for(def city : citys){
def data = CcbCommonUtils.findCityCodeValue(city)
def data = CcbCommonUtils.findCityCodeValue()
def map = [:]
map.name = city
if(data){
@@ -36,23 +36,38 @@ class EvaluateRateBatch extends BaseBatch{
def createDatas(def datas){
def allRows = []
datas.each { m ->
def data = []
data << m.id.toString()
data << m.rate_type
data << m.rate
data << m.rank
data << "È«¹ú"
data << m.rank_remark
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW"
allRows << data
if(m.rank_remark!="开发商装修"){
def data = []
data << m.id.toString()
data << m.rate_type
data << m.rate
data << m.rank
data << "全国"
data << converFitment(m.rank_remark)
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
data << "FJW"
allRows << data
}
}
return allRows
}
private def converFitment(rank_remark){
if(rank_remark){
if(rank_remark.contains("简单装修")){
return "简装"
}else if(rank_remark.contains("精装修")){
return "精装"
}else if(rank_remark.contains("豪华装修")){
return "豪装"
}
}
return rank_remark
}
private def findEvaluationRate(){
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM evaluation_rate WHERE TYPE = 'Ð¡Çø'";
def sql = "SELECT id, rate_type, rate, rank, rank_remark,date_created, last_updated FROM rate WHERE TYPE = '小区'";
def rets = []
sqlInstance.eachRow(sql,{
rets << ["id":it.id, "rate_type":it.rate_type, "rate":it.rate, "rank":it.rank,"rank_remark":it.rank_remark,"c_date":it.date_created, "m_date":it.last_updated]
@@ -31,7 +31,7 @@ class GuiDingPriceBatch extends BaseBatch{
def rows = []
for (def city : citys) {
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def prices = CcbCommonUtils.findPersonPrice(city, DateUtils.addMonth(new Date(), -6))
@@ -61,25 +61,24 @@ class GuiDingPriceBatch extends BaseBatch{
def dist = distMap.get(ps.region+"##"+ps.name)
if(dist){
def ret = []
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.city_code
ret << cityCodeDatas.fangjia_city
if(dist.region){
ret << distCodeMap.get(dist.region)
}else{
def areaData = distCodeMap.get(dist.region)
if(areaData){
ret << ps._id.toString()
ret << dist._id.toString()
ret << dist.name
ret << cityCodeDatas.city_code
ret << cityCodeDatas.fangjia_city
ret << areaData.district_code
ret << areaData.district
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << ps.guiding_person
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
ret << dist.region
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << ps.guiding_value
ret << ps.guiding_person
ret << ""
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
}
@@ -87,7 +86,7 @@ class GuiDingPriceBatch extends BaseBatch{
}
def getDistrictMap(def city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
def districts = CcbCommonUtils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
def distMap = [:]
for(def d : districts){
distMap.put(d.region+"##"+d.name, d)
@@ -27,7 +27,7 @@ class StagingDistrictBatch extends BaseBatch{
def citys = CcbCommonUtils.citys;
def rows = []
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "master_district_name", "c_date", "m_date"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2","name_cat","lng","master_district_name", "c_date", "m_date"])
def fenQiMap = getFenQiMap(dists);
def rets = createCityData(dists, fenQiMap)
rows.addAll(rets)
@@ -40,8 +40,8 @@ class StagingDistrictBatch extends BaseBatch{
results << headers
results.addAll(rows)
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\staging_district.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\staging_district.dat",headers, "utf-8");
ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\staging_district.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\staging_district.txt",headers, "utf-8", false);
}
@@ -76,9 +76,9 @@ class StagingDistrictBatch extends BaseBatch{
def getFenQiMap(def dists){
def fenQiMap = [:]
for(def d : dists){
if(d.master_district_name){
// if(d.master_district_name){
fenQiMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// }
}
return fenQiMap;
}
@@ -1,21 +1,355 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import org.bson.types.ObjectId
import java.util.regex.*;
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def citys = CcbCommonUtils.citys;
def rows = []
DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
for(def city : citys) {
def data = CcbCommonUtils.findCityCodeValue(city)
def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'";
//def citys = CcbCommonUtils.citys;
//def rows = []
//
//for(def city : citys) {
// def data = CcbCommonUtils.findCityCodeValue()
// def sql = "SELECT count(*) as count FROM bigdata_forsale WHERE cityid = '${data.city_code}'";
//
// def count = 0
// sqlInstance.execute(sql,{
// count = it.count
// });
// println city +" "+count
//}
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0804(3).xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\7月成交.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
// dealMap.put(d['A'], d['B'])
//}
//
//
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['C']){
// d['Q'] = dealMap.get(d['C'])
// }
//},2)
def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807 - 副本.xlsx"
//def datas = ExcelUtils.read("d:\\数据\\建行\\房产估值\\小区字段.xlsx",0,1)
//def dealMap = [:]
//for(def d : datas){
// def map = [:]
// map.put("land_area",d['D'])
// map.put("build_area",d['F'])
// map.put("greeningrate_area",d['H'])
// map.put("parkingcnt_area",d['J'])
// map.put("zts_area",d['L'])
// map.put("buildyear_area",d['N'])
// map.put("plotratio",d['P'])
//
// dealMap.put(d['A'], map)
//}
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
//小区量
//ExcelUtils.multiThreadingModify(fileName, { d ->
// def city = d['B']
// if (city) {
// def cityid = cityCodeDatas.get(city)
// def total = findCount(cityid, sqlInstance)
// d['K'] = total
// }
//},0)
//小区字段覆盖率
ExcelUtils.multiThreadingModify(fileName,{d->
def city = d['A']
if(city){
def cityid = cityCodeDatas.get(city)
def total = findCount(cityid, sqlInstance)
d['AA'] = total
def a = findEstateType(cityid, sqlInstance)
d['AB'] = DataHandleUtils.getDoubleRound(2, a*100/total)
def b = findFiledCount(cityid, 'buildyear', sqlInstance)
d['AC'] = DataHandleUtils.getDoubleRound(2, b*100/total)
def c = findFiledCount(cityid, 'address', sqlInstance)
d['AD'] = DataHandleUtils.getDoubleRound(2, c*100/total)
def d1 = findFiledCount(cityid, 'ZTS', sqlInstance)
d['AE'] = DataHandleUtils.getDoubleRound(2, d1*100/total)
def e = findFiledCount(cityid, 'landarea', sqlInstance)
d['AF'] = DataHandleUtils.getDoubleRound(2, e*100/total)
def f = findFiledCount(cityid, 'buildarea', sqlInstance)
d['AG'] = DataHandleUtils.getDoubleRound(2, f*100/total)
def g = findFiledCount(cityid, 'plotratio', sqlInstance)
d['AH'] = DataHandleUtils.getDoubleRound(2, g*100/total)
def h = findFiledCount(cityid, 'greeningrate', sqlInstance)
d['AI'] = DataHandleUtils.getDoubleRound(2, h*100/total)
def i = findFiledCount(cityid, 'latitude', sqlInstance)
d['AJ'] = DataHandleUtils.getDoubleRound(2, i*100/total)
def j = findFiledCount(cityid, 'parkingcnt', sqlInstance)
d['AK'] = DataHandleUtils.getDoubleRound(2, j*100/total)
println d
}
},1)
//def fileName = "d:\\数据\\建行\\房产估值\\采购数据交付标准_0807-1.xlsx"
//def dealMap = findDealCount(sqlInstance)
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(d['C']){
// d['Q'] = dealMap.get(d['C'])
// }
//},2)
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
//def updateIndex = 0
//def deleteIndex = 0
//ExcelUtils.modify(fileName,{d->
// if(d['R'] && d['R'].toString().toDouble().toInteger()==1){
// def dist = CommonDatautils.findDistrictById(d['D'])
// def city = dist.city
// def region = RegionData.formatRegion(city, d['Q'])
// if(region && dist.region!=region){
// println city +" new:"+region +" old:"+d['C']
// def udata = new BasicDBObject()
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){
// deleteIndex++
// udata.put("status", 1)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
// println deleteIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
// }else{
// updateIndex++
// udata.put("region", region)
// udata.put("m_date", new Date())
// DbUtils.seaweed.update(new BasicDBObject('_id',dist._id),new BasicDBObject('$set', udata))
// println updateIndex +" "+dist.city +" "+ dist.name+" "+dist.region+"("+region+")"
// }
// }else{
// println d['D']+" 不存在"
// }
// }
//},0)
def findDealCount(sqlInstance){
def sql = "select cityname,count(*) as count from bigdata_deal where dealdate>=to_date('2021-06-01','yyyy-mm-dd') and dealdate<to_date('2021-07-01','yyyy-mm-dd') group by cityname";
def count = 0
sqlInstance.execute(sql,{
count = it.count
def map = [:]
sqlInstance.eachRow(sql,{r->
map.put(r.cityname, r.count)
});
println city +" "+count
return map
}
def findEstateType(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_commbuildtype where cityid='${cityid}' and buildtype!='0199'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findCount(cityid, sqlInstance){
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findFiledCount(cityid, field, sqlInstance){
def sql = "select count(*) as count from bigdata_community where cityid='${cityid}' and "+field+" is not null";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
//def fileName = "d:\\数据\\建行\\房产估值\\经纬度错误数据修正.xlsx"
//def updateIndex = 0
//def delIndex = 0
//def cityMap = [:]
//ExcelUtils.multiThreadingModify(fileName,{d->
// if(!d['R']){
//
// def obj = CommonDatautils.findDistrictById(d['D'])
// if(obj){
// def ranges = cityMap.get(obj.city)
// if(!ranges){
// def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",obj.city))
// ranges = []
// for(def r : dbcursor){
// ranges << r
// }
// cityMap.put(obj.city, ranges)
// }
//
// if(ranges){
// def lng = obj.lng/10000000000.0
// def lat = obj.lat/10000000000.0
//
// def region = null
// for(def r : ranges){
// try{
// def flag = isInArea(lng, lat, r.polyline)
// if(flag){
// def regionStr = r.region
// region = RegionData.formatRegion(obj.city, regionStr)
// break
// }
// }catch(Exception e){
// println e.getMessage()
// }
//
// }
// if(region && region!=obj.region){
// updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",obj.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]old:${obj.region} new:"+region+" "+obj.name
// }else if(region && region==obj.region){
// println "[same] "+obj.city +" "+obj.name
// }else{
// DbUtils.seaweed.update(new BasicDBObject("_id",new ObjectId(d['D'])),new BasicDBObject('$set', new BasicDBObject("status", 1)))
// delIndex++
// println delIndex +" [delete] "+obj.city +" "+obj.name
// }
// }else{
// println obj.city +" 没有此城市边框信息"
// }
//
// }else{
// println d['D']+" 没找到该小区"
// }
//
// }
//},0);
//def city = "杭州"
//def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lat", "lng"])
//def dbcursor = POLYLINECOLL.find(new BasicDBObject("city",city))
//def ranges = []
//for(def d : dbcursor){
// ranges << d
//}
//def index =0
//def updateIndex = 0
//def delIndex = 0
//for(def dist : dists){
//// if(dist.region=="江干"){
// def lat = dist.lat/10000000000.0
// def lng = dist.lng/10000000000.0
//
// def regionStr =null
// def region = null
// for(def r : ranges){
// def flag = isInArea(lng, lat, r.polyline)
// if(flag){
// regionStr = r.region
// region = RegionData.formatRegion(dist.city, regionStr)
// break
// }
// }
// // def regionStr = getStreetInfo(city, lng.toString()+","+lat.toString())
// if(regionStr && region!=dist.region){
//
// def obj = CommonDatautils.findDistrict(dist.city, region, dist.name, "district")
// if(obj){
// delIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("status", 1)))
// println delIndex+" [del]"+region +" "+dist.name
// }else{
// updateIndex++
// DbUtils.seaweed.update(new BasicDBObject("_id",dist.get("_id")),new BasicDBObject('$set', new BasicDBObject("region", region)))
// println updateIndex +" [update]old:${dist.region} new:"+region+" "+dist.name
// }
// }else{
// index++
// println index+" [没找到]"+dist.name
// }
//// }
//}
def isInArea(lng, lat, points){
List<Point> pointList = InsidePolygon.createPoint2(points);
//Point point = new Point(lng, lat);
//def flag = InsidePolygon.aboutZone(point, pointList)
def flag = InsidePolygon.isPtInPoly(lng, lat, pointList)
return flag;
}
def getStreetInfo(def city, def glocation){
def township = null
if(glocation){
def jsonObject = GaoDeApiUtils.geocode(glocation)
if(jsonObject && jsonObject.regeocode) {
def addressComponent = jsonObject.regeocode.addressComponent
if(addressComponent && addressComponent.city.contains(city)){
township = addressComponent.township
}
}
}
return township;
}
@@ -18,8 +18,8 @@ import com.mongodb.DBCollection
@BatchOwner(Watcher.GaoJuan)
class ThirdPartyPriceBatch extends BaseBatch {
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM"));
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
@@ -35,17 +35,25 @@ class ThirdPartyPriceBatch extends BaseBatch {
def index = 0
for(def city : citys){
index++
def cityCodeDatas = CcbCommonUtils.findCityCodeValue(city)
def cityCodeDatas = CcbCommonUtils.findCityCodeValue()
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name"])
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name","name_cat", "estate_type2","lng"])
def thirdPriceMap = findThirdPartyPrice(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, dists, thirdPriceMap)
rows.addAll(rets)
println index+" "+city+" "+rets.size()
}
def fileName = "/home/goojia/ccb/third_party_price.txt"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt"
}
def header = ["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"]
FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\third_party_price.txt", header, "utf-8", false);
FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false);
}
@@ -55,103 +63,102 @@ class ThirdPartyPriceBatch extends BaseBatch {
for(def d : dists){
def third = thirdPriceMap.get(d.region+"##"+d.name)
if(third){
def ret = []
ret << third._id.toString()
ret << d._id.toString()
ret << d.name
ret << cityCodeDatas.city_code
ret << cityCodeDatas.fangjia_city
if(d.region){
ret << distCodeMap.get(d.region)
}else{
ret << ""
}
ret << d.region
ret << third.s_date
def data = distCodeMap.get(d.region)
if(data){
def ret = []
ret << third.id.toString()
ret << d._id.toString()
ret << d.name
ret << cityCodeDatas.get(d.city)
ret << d.city
ret << data.district_code
ret << data.district
ret << sDate
if(third.zhugezhaofang_esf_price){
ret << third.zhugezhaofang_esf_price
}else{
ret << ""
}
if(third.anjuke_esf_price){
ret << third.anjuke_esf_price
}else{
ret << ""
}
if(third.soufang_esf_price){
ret << third.soufang_esf_price
}else{
ret << ""
}
if(third.zhugezhaofang_esf_price){
ret << third.zhugezhaofang_esf_price
}else{
ret << ""
}
if(third.anjuke_esf_price){
ret << third.anjuke_esf_price
}else{
ret << ""
}
if(third.soufang_esf_price){
ret << third.soufang_esf_price
}else{
ret << ""
}
if(third.lianjia_esf_price){
ret << third.lianjia_esf_price
}else{
ret << ""
}
if(third.lianjia_esf_price){
ret << third.lianjia_esf_price
}else{
ret << ""
}
if(third.cityhouse_esf_price){
ret << third.cityhouse_esf_price
}else{
ret << ""
}
if(third.cityhouse_esf_price){
ret << third.cityhouse_esf_price
}else{
ret << ""
}
if(third.maitian_esf_price){
ret << third.maitian_esf_price
}else{
ret << ""
}
if(third.maitian_esf_price){
ret << third.maitian_esf_price
}else{
ret << ""
}
if(third.beike_esf_price){
ret << third.beike_esf_price
}else{
ret << ""
}
if(third.beike_esf_price){
ret << third.beike_esf_price
}else{
ret << ""
}
if(third.get("365house_esf_price")){
ret << third.get("365house_esf_price")
}else{
ret << ""
}
if(third.get("365house_esf_price")){
ret << third.get("365house_esf_price")
}else{
ret << ""
}
if(third.zhongyuan_esf_price){
ret << third.zhongyuan_esf_price
}else{
ret << ""
}
if(third.zhongyuan_esf_price){
ret << third.zhongyuan_esf_price
}else{
ret << ""
}
if(third.tuboshi_esf_price){
ret << third.tuboshi_esf_price
}else{
ret << ""
}
if(third.qfang_esf_price){
ret << third.qfang_esf_price
}else{
ret << ""
}
if(third.tuboshi_esf_price){
ret << third.tuboshi_esf_price
}else{
ret << ""
}
if(third.qfang_esf_price){
ret << third.qfang_esf_price
}else{
ret << ""
}
if(third.fangtan_esf_price){
ret << third.fangtan_esf_price
}else{
ret << ""
}
if(third.fangtan_esf_price){
ret << third.fangtan_esf_price
}else{
ret << ""
}
if(third.fangduoduo_esf_price){
ret << third.fangduoduo_esf_price
}else{
ret << ""
if(third.fangduoduo_esf_price){
ret << third.fangduoduo_esf_price
}else{
ret << ""
}
if(third.get("5i5j_esf_price")){
ret << third.get("5i5j_esf_price")
}else{
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
if(third.get("5i5j_esf_price")){
ret << third.get("5i5j_esf_price")
}else{
ret << ""
}
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << "FJW"
rets << ret
}
}
@@ -162,13 +169,83 @@ class ThirdPartyPriceBatch extends BaseBatch {
def findThirdPartyPrice(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("s_date", curDate)
def dbcursor = AVERAGE_STATISTICS.find(q)
q.put("s_date", new BasicDBObject('$gte', sDate))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1))
def map = [:]
for(def db : dbcursor){
map.put(db.region+"##"+db.name, db)
def key = db.region+"##"+db.name
def dataMap = map.get(key)
if(!dataMap){
dataMap = [:]
dataMap.put("id", db._id.toString())
map.put(key, dataMap)
}
if(!dataMap.containsKey("zhugezhaofang_esf_price") && db.zhugezhaofang_esf_price && db.zhugezhaofang_esf_price>3000){
dataMap.put("zhugezhaofang_esf_price", db.zhugezhaofang_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){
dataMap.put("anjuke_esf_price", db.anjuke_esf_price)
}
if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){
dataMap.put("anjuke_esf_price", db.get("58tc_esf_price"))
}
if(!dataMap.containsKey("soufang_esf_price") && db.soufang_esf_price && db.soufang_esf_price>3000){
dataMap.put("soufang_esf_price", db.soufang_esf_price)
}
if(!dataMap.containsKey("lianjia_esf_price") && db.lianjia_esf_price && db.lianjia_esf_price>3000){
dataMap.put("lianjia_esf_price", db.lianjia_esf_price)
}
if(!dataMap.containsKey("cityhouse_esf_price") && db.cityhouse_esf_price && db.cityhouse_esf_price>3000){
dataMap.put("cityhouse_esf_price", db.cityhouse_esf_price)
}
if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){
dataMap.put("maitian_esf_price", db.maitian_esf_price)
}
if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){
dataMap.put("beike_esf_price", db.beike_esf_price)
}
if(!dataMap.get("365house_esf_price") && db.get("365house_esf_price") && db.get("365house_esf_price")>3000){
dataMap.put("365house_esf_price", db.get("365house_esf_price"))
}
if(!dataMap.containsKey("zhongyuan_esf_price") && db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){
dataMap.put("zhongyuan_esf_price", db.zhongyuan_esf_price)
}
if(!dataMap.containsKey("tuboshi_esf_price") && db.tuboshi_esf_price && db.tuboshi_esf_price>3000){
dataMap.put("tuboshi_esf_price", db.tuboshi_esf_price)
}
if(!dataMap.containsKey("qfang_esf_price") && db.qfang_esf_price && db.qfang_esf_price>3000){
dataMap.put("qfang_esf_price", db.qfang_esf_price)
}
if(!dataMap.containsKey("fangtan_esf_price") && db.fangtan_esf_price && db.fangtan_esf_price>3000){
dataMap.put("fangtan_esf_price", db.fangtan_esf_price)
}
if(!dataMap.containsKey("fangduoduo_esf_price") && db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){
dataMap.put("fangduoduo_esf_price", db.fangduoduo_esf_price)
}
if(!dataMap.get("5i5j_esf_price") && db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){
dataMap.put("5i5j_esf_price", db.get("5i5j_esf_price"))
}
// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){
// map.put(key, db)
// }
}
return map
def resultMap = [:]
map.each {key, value->
if(value && value.size()>1){
resultMap.put(key, value)
}
}
return resultMap
}
@@ -0,0 +1,294 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
/***
* 建行成交抓取统计
*/
class TradeStatLog {
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com")
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new TradeStatLog()
//task.run()
task.thirdPartyStat()
// println Calendar.getInstance().get(Calendar.YEAR)
}
def run(){
def citys = CcbCommonUtils.citys;
def rets = []
rets << ["城市","5月总量","6月总量","贝壳","链家", "我爱我家","房天下","中原"]
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
def startDate = "2021-05-01"
def date = "2021-06-01"
def endDate = "2021-07-01"
for(def city : citys){
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def curMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def ret = []
ret << city
if(city == "上海"){
def lastMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def curMonthYoudaCount = getYouDaData(DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
def a = lastMonthCount.toInteger() + lastMonthYoudaCount.toInteger()
def b = curMonthCount.toInteger() + curMonthYoudaCount.toInteger()
ret << a
ret << b
def c = (b-a)*100/a
def pre = DataHandleUtils.getDoubleRound(2, c)
ret << pre
}else{
ret << lastMonthCount
ret << curMonthCount
if(lastMonthCount){
def a = (curMonthCount-lastMonthCount)*100/lastMonthCount
def pre = DataHandleUtils.getDoubleRound(2, a)
ret << pre
}else{
ret << null
}
}
for(def source : sources){
def sourceLastMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
def sourceCurMonthCount = findTradesCount(city, source, DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd"))
ret << sourceLastMonthCount
ret << sourceCurMonthCount
}
rets << ret
println ret
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\成交统计.xlsx", rets, "数据")
println "表格生成完成"
}
def thirdPartyStat(){
def citys = CcbCommonUtils.citys;
def rets = []
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
rets << ["城市","小区总量","有第三方的总量","1家占比","2家占比","3家占比","4家占比","5家占比","6家占比","7家占比","8家占比","9家占比","10家占比","11家占比","12家占比","13家占比","14家占比"]
for(def city : citys){
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "estate_type2", "name_cat","lng"]);
def m = getThirdPartyPricePrecent(city)
rets << [city, districts.size(), m.total, m.count1, m.count2, m.count3, m.count4, m.count5, m.count6, m.count7, m.count8, m.count9, m.count10, m.count11, m.count12, m.count13, m.count14]
println rets.get(rets.size()-1)
}
ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\第三方均价统计.xlsx", rets, "数据")
println "表格生成完成"
}
public static def findTradesCount(city, source, startDate, endDate){
def q = new BasicDBObject()
q.put("city", city)
if(source){
q.put("source", source)
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def count = DEAL_PRICE_COLL.count(q)
return count
}
def getYouDaData(startDate, endDate){
def q = new BasicDBObject()
q.put("contractTime", new BasicDBObject('$gte', startDate).append('$lt', endDate));
def count = youda_esfhouse_coll.count(q)
return count;
}
def getThirdPartyPricePrecent(def city){
def sql = "SELECT * FROM third_party_price WHERE cityname = '"+city+"'";
def count0 = 0
def count1 = 0
def count2 = 0
def count3 = 0
def count4 = 0
def count5 = 0
def count6 = 0
def count7 = 0
def count8 = 0
def count9 = 0
def count10 = 0
def count11 = 0
def count12 = 0
def count13 = 0
def count14 = 0
def total = 0
sqlInstance.eachRow(sql,{third->
total++
def ret = []
if(third.zhuge_avg_price){
ret << third.zhuge_avg_price
}
if(third.anjuke_avg_price){
ret << third.anjuke_avg_price
}
if(third.fang_avg_price){
ret << third.fang_avg_price
}
if(third.lianjia_avg_price){
ret << third.lianjia_avg_price
}
if(third.cityhouse_avg_price){
ret << third.cityhouse_avg_price
}
if(third.maitian_avg_price){
ret << third.maitian_avg_price
}
if(third.ke_avg_price){
ret << third.ke_avg_price
}
if(third.taofang_avg_price){
ret << third.taofang_avg_price
}
if(third.zhongyuan_avg_price){
ret << third.zhongyuan_avg_price
}
if(third.tbs_avg_price){
ret << third.tbs_avg_price
}
if(third.qfang_avg_price){
ret << third.qfang_avg_price
}
if(third.fangtan_avg_price){
ret << third.fangtan_avg_price
}
if(third.fangduoduo_avg_price){
ret << third.fangduoduo_avg_price
}
if(third.wiwj_avg_price){
ret << third.wiwj_avg_price
}
if(ret.size()==0){
count0++
}
if(ret.size()==1){
count1++
}
if(ret.size()==2){
count2++
}
if(ret.size()==3){
count3++
}
if(ret.size()==4){
count4++
}
if(ret.size()==5){
count5++
}
if(ret.size()==6){
count6++
}
if(ret.size()==7){
count7++
}
if(ret.size()==8){
count8++
}
if(ret.size()==9){
count9++
}
if(ret.size()==10){
count10++
}
if(ret.size()==11){
count11++
}
if(ret.size()==12){
count12++
}
if(ret.size()==13){
count13++
}
if(ret.size()==14){
count14++
}
});
def pre1 = 0
def pre2 = 0
def pre3 = 0
def pre4 = 0
def pre5 = 0
def pre6 = 0
def pre7 = 0
def pre8 = 0
def pre9 = 0
def pre10 = 0
def pre11 = 0
def pre12 = 0
def pre13 = 0
def pre14 = 0
if(total){
pre1 = DataHandleUtils.getDoubleRound(2, count1*100.0/total)
pre2 = DataHandleUtils.getDoubleRound(2, count2*100.0/total)
pre3 = DataHandleUtils.getDoubleRound(2, count3*100.0/total)
pre4 = DataHandleUtils.getDoubleRound(2, count4*100.0/total)
pre5 = DataHandleUtils.getDoubleRound(2, count5*100.0/total)
pre6 = DataHandleUtils.getDoubleRound(2, count6*100.0/total)
pre7 = DataHandleUtils.getDoubleRound(2, count7*100.0/total)
pre8 = DataHandleUtils.getDoubleRound(2, count8*100.0/total)
pre9 = DataHandleUtils.getDoubleRound(2, count9*100.0/total)
pre10 = DataHandleUtils.getDoubleRound(2, count10*100.0/total)
pre11 = DataHandleUtils.getDoubleRound(2, count11*100.0/total)
pre12 = DataHandleUtils.getDoubleRound(2, count12*100.0/total)
pre13 = DataHandleUtils.getDoubleRound(2, count13*100.0/total)
pre14 = DataHandleUtils.getDoubleRound(2, count14*100.0/total)
}
def map = [:]
map.total = total
map.count1 = pre1
map.count2 = pre2
map.count3 = pre3
map.count4 = pre4
map.count5 = pre5
map.count6 = pre6
map.count7 = pre7
map.count8 = pre8
map.count9 = pre9
map.count10 = pre10
map.count11 = pre11
map.count12 = pre12
map.count13 = pre13
map.count14 = pre14
return map
}
}
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name","东寓新村")
// q.put("name","½ð»Ô²®¾ôɽ")
DBObject keys = new BasicDBObject();
if(fields){
@@ -698,6 +698,28 @@ class CommonDatautils {
return rets;
}
public static def findDistrictByCityAndRegions(def city, def cat, def regions, def fields){
DBObject q = new BasicDBObject();
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
q.put("region", new BasicDBObject('$in', regions))
DBObject keys = new BasicDBObject();
if(fields){
for(def field : fields){
keys.put(field, 1);
}
}
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
rets << d
}
return rets;
}
public static def findAllDistrictByQuery(def q, def cat, def fields){
@@ -758,6 +780,8 @@ class CommonDatautils {
keys.put("address", 1);
keys.put("build_structure", 1);
keys.put("out_pics", 1);
keys.put("lat", 1);
keys.put("lng", 1);
DSCursor cursor = getDataStoreConnection().find(q, keys);
if(cursor){
for(def obj : cursor){
@@ -1292,7 +1316,7 @@ class CommonDatautils {
}
public static def getAllDictionary(){
def dbCursor = DICT_COLL.find(new BasicDBObject())
def dbCursor = DICT_COLL.find(new BasicDBObject()).batchSize(5000)
def rets = []
def filters = [ "estate","hfword","main","metro","station"]
for(def obj : dbCursor){
@@ -14,7 +14,7 @@ import com.mongodb.DBCollection
//福州第三方均价统计,发送钉钉消息
//def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1)
//def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv");
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202106.csv")
def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202107.csv")
def count0 = 0
def count1 = 0
def count2 = 0
@@ -1,11 +1,16 @@
package com.fangjia.script.gj
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
@@ -63,7 +68,7 @@ class AddThirdPriceDistrict {
def resAddrMap = [:]
def filterEstateTypeMap = [:]
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -82,14 +87,78 @@ class AddThirdPriceDistrict {
// println mapName.replaceAll("(\\w+,)?+\\w+(幢)", "")
}
def run(){
exportDatas();
// getDataType("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
// addInfo();
// addMapInfo("D:\\数据\\小区\\苏州第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx")
// exportDatas();
// getDataType("D:\\数据\\小区\\南昌第三方均价.xlsx");
// addInfo("D:\\数据\\小区\\南昌第三方均价.xlsx");
// addMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// addCrawlMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// dataFilter("D:\\数据\\小区\\南昌第三方均价.xlsx")
// matchDistDatas("D:\\数据\\小区\\南昌第三方均价.xlsx")
addNewMapInfo("D:\\数据\\小区\\南昌第三方均价.xlsx")
// addFlag("苏州","D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx");
}
def dataFilter(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
def name = row['C']
if(name){
def flag = NameProcess.filterName(name)
if(!flag){
row['T'] = 1
}else{
if(name.toString().length()<3){
if(row['N'] && row['N']>=10 && !row['J']){
row['T'] = 25
}else{
row['T'] = 18
}
}else{
def s1 = com.fangjia.core.util.DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
if(s1){
row['T'] = 19
}
}
}
}else{
if(!name){
row['T'] = 1
}
}
});
println "3.名称过滤完成"
}
def addNewMapInfo(def path){
ExcelUtils.multiThreadingModify(path, { row ->
if(!row['I']){
def gdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "高德")
if(gdData){
row['F'] = gdData.district
row['G'] = gdData.name
row['H'] = gdData.address
row['I'] = gdData.type
row['J'] = gdData.score
row['K'] = gdData.lng
row['L'] = gdData.lat
}
def bdData = FangjiaClientUtils.getMapData(row['A'], row['B'], row['C'], row['D'],"", "百度")
if(bdData){
row['M'] = bdData.district
row['N'] = bdData.name
row['O'] = bdData.address
row['P'] = bdData.type
row['Q'] = bdData.score
row['R'] = bdData.lng
row['S'] = bdData.lat
}
}
},0);
}
def addCrawlMapInfo( def path){
ExcelUtils.multiThreadingModify(path, { row ->
def flag = row['R']
@@ -105,27 +174,27 @@ class AddThirdPriceDistrict {
row['K'] = data.lat
}
def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
def bdRegion =null
def bdName = null
def bdAddress = null
def bdLng = null
def bdLat = null
def bdEstateType = null
if(baiduRets){
bdRegion = baiduRets.region
bdName = baiduRets.name
bdAddress = baiduRets.address
bdLng = baiduRets.lng
bdLat = baiduRets.lat
bdEstateType = baiduRets.estateType
}
row['L'] = bdRegion
row['M'] = bdName
row['N'] = bdAddress
row['O'] = bdEstateType
row['P'] = bdLng
row['Q'] = bdLat
// def baiduRets = findBaiduMap(row['A'], row['B'], row['C'])
// def bdRegion =null
// def bdName = null
// def bdAddress = null
// def bdLng = null
// def bdLat = null
// def bdEstateType = null
// if(baiduRets){
// bdRegion = baiduRets.region
// bdName = baiduRets.name
// bdAddress = baiduRets.address
// bdLng = baiduRets.lng
// bdLat = baiduRets.lat
// bdEstateType = baiduRets.estateType
// }
// row['L'] = bdRegion
// row['M'] = bdName
// row['N'] = bdAddress
// row['O'] = bdEstateType
// row['P'] = bdLng
// row['Q'] = bdLat
}
},0);
@@ -502,8 +571,8 @@ class AddThirdPriceDistrict {
return thirdPartyColl.findOne(q)
}
def addInfo(){
def addInfo(filePath){
province = "江西"
// def city = "苏州"
// def datas = findMatchErrorDatas(city);
// println "共【"+datas.size()+"】条"
@@ -512,8 +581,8 @@ class AddThirdPriceDistrict {
// def resMap = groupByName(datas);
def index = 0;
ExcelUtils.multiThreadingModify("D:\\数据\\小区\\苏州第三方均价(1)(2).xlsx", { row ->
def flag = row['R']
ExcelUtils.multiThreadingModify(filePath, { row ->
def flag = row['T']
def dgDatas = row['G']
// if(!flag){
index++
@@ -547,9 +616,9 @@ class AddThirdPriceDistrict {
// def estateType = filterEstateTypeMap.get(row['A']+"##"+row['B']+"##"+row['C'])
// row['W'] = estateType
if(!row['X']){
def name = nameProcess(row['B'], row['C'])
row['X'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C'])
if(!row['Z']){
//def name = nameProcess(row['B'], row['C'])
row['Z'] = EsSearchUtils.findHouse(row['A'], row['B'] , row['C'])
}
//
@@ -956,11 +1025,11 @@ class AddThirdPriceDistrict {
def queryPoiMap(def city, def region, def name){
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT cityname,adname,name,address,type,location.lon,location.lat,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1")
sqlBuffer.append(" and cityname = '" + city + "'")
sqlBuffer.append(" and adname = '" + region + "'")
sqlBuffer.append(" and name = '" + name + "'")
sqlBuffer.append(" and source = '高德'")
sqlBuffer.append(" and MATCH(name,'${name}')")
sqlBuffer.append(" and source = '高德' order by score() desc limit 1")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def map = [:]
while (rs.next()) {
@@ -969,8 +1038,11 @@ class AddThirdPriceDistrict {
map.name = rs.getString("name")
map.address = rs.getString("address")
map.type = rs.getString("type")
map.lng = rs.getString("location.lon")
map.lat = rs.getString("location.lat")
def location = rs.getObject("location")
if(location){
map.lng = location.x
map.lat = location.y
}
}
return map;
}
@@ -985,6 +1057,37 @@ class AddThirdPriceDistrict {
return null
}
def matchDistDatas(def path){
def index = 0
ExcelUtils.multiThreadingModify(path,{ List row ->
index ++
if(!row['I'] && !row['J']){
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['C'], row['D']);
if(rMap && (rMap.region==row['B'] || row['B'].contains(rMap.region))){
//row['J'] = rMap.city
row['U'] = rMap.region
row['V'] = rMap.name
row['W'] = rMap.address
if(rMap.lat){
// def baidu = BaiDuApiUtils.geoconv(row['F'].toDouble(), row['G'].toDouble(), 3)
// def maplng = baidu.x
// def maplat = baidu.y
// row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
def score = CosineSimilarAlgorithm.getSimilarity(row['C'], row['V'])
if(score<0.3){
score = CosineSimilarAlgorithm.getSimilarity(row['D'], row['V'])
}
row['X'] = score
}
}
println index +" "+row
}
},0)
println "5.小区匹配完成"
}
}
@@ -64,24 +64,29 @@ class EsSearchUtils {
sqlBuffer.append(" and region = '"+region+"'")
sqlBuffer.append(" and district_name = '"+name+"'")
sqlBuffer.append(" and cat = 'sell'")
sqlBuffer.append(" limit 100")
sqlBuffer.append(" limit 200")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def rets = []
while (rs.next()){
def height = rs.getInt("height")
def direction = rs.getString("direction")
def fitment = rs.getString("fitment")
def room = rs.getInt("room")
def hall = rs.getInt("hall")
def toilet = rs.getInt("toilet")
def map = [:]
map.height = height
map.direction = direction
map.fitment = fitment
map.room = room
map.hall = hall
map.toilet = toilet
rets << map
if(room){
def height = rs.getInt("height")
def direction = rs.getString("direction")
def fitment = rs.getString("fitment")
def hall = rs.getInt("hall")
def toilet = rs.getInt("toilet")
def area = rs.getInt("area")
def map = [:]
map.height = height
map.direction = direction
map.fitment = fitment
map.room = room
map.hall = hall
map.toilet = toilet
map.area = area
rets << map
}
}
return rets;
}
@@ -259,6 +259,31 @@ class FiledStrand {
return updateFile;
}
static def processArea(all_area){
if(all_area){
def v = getValue(all_area)
if(v>=1000 && v<=10000000){
return v.toString()
}
}
return null
}
static def processCarSite(car_site_count){
def v = getValue(car_site_count.toString());
if(v){
if(v>=1 && v<=100000){
try {
return v.toString().toInteger();
}catch(Exception e){
println e.getMessage()
}
}
}
return null
}
static def addBuildAreaToData(def dist, def updateFile){
def build_area = null;
if(dist.build_area){
@@ -298,6 +323,26 @@ class FiledStrand {
}
return updateFile;
}
public static def processCubagePercent(cubage_percent, estate_type2){
if(cubage_percent){
def v = getValue(cubage_percent)
if(v){
if(estate_type2 == "ąđĘű"){
if(v>=0.3 && v<=1){
return v.toString()
}
}else{
if(v>1 && v<=10){
return v.toString()
}
}
}
}
return null
}
static def addVirescencePercentToData(dist, updateFile){
def virescence_percent = null
if(dist.virescence_percent){
@@ -312,6 +357,16 @@ class FiledStrand {
return updateFile;
}
static def processVirescencePercent(virescence_percent){
if(virescence_percent){
def v = getValue(virescence_percent)
if(v && v>15 && v<=70){
return v.toString()
}
}
return null;
}
static def addBuildTypeToData(dist, updateFile){
if(dist.estateType2 && dist.buildType){
def house = ["超高层","小高层","多层","高层","花园洋房","老洋房","四合院"]
@@ -50,14 +50,14 @@ class GaoDeMapCrawl extends BaseBatch{
// def categorys = ["141202", "141203"]//中学,小学
// def prefix = "district_"
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
def prefix = "district_"
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def prefix = "road_"
// def categorys = ["190301"]//道路相关
def prefix = "village_"
def categorys = ["190108","190109"]//村庄
// def prefix = "village_"
// def categorys = ["190108","190109"]//村庄
// def prefix = "company_"
// def categorys = ["170000","170100","170200","170201","170202","170203","170204","170205","170206","170207","170208","170209","170300","170400","170401","170402","170403","170404","170405","170406","170407","170408"]
@@ -119,7 +119,7 @@ class GaoDeMapCrawl extends BaseBatch{
def run(){//,"西安","重庆"
//def citys = ["上海","重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def citys =["无锡"]
def citys =["南昌"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
@@ -1,6 +1,7 @@
package com.fangjia.script.gj
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.commData.CommonDatautils
@@ -28,8 +29,9 @@ class RegionPointCrawl {
static main(args) {
def task = new RegionPointCrawl();
//河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
// def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"]
//def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"]
def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"]
for(def p : provinces){
task.crawlCityAndDistrictCode(p);
task.run(p);
@@ -55,7 +57,13 @@ class RegionPointCrawl {
}
println index +" "+cityMap.name+" "+m.name+" "+ret.size()
if(ret.size()>0){
saveData(cityMap.name, m.name, m.ad_code , ret);
def standardCity = RegionData.formatCity(cityMap.name)
if(standardCity){
saveData(standardCity, m.name, m.ad_code , ret);
}else{
println cityMap.name +" 没有标准化"
}
}
}
}
@@ -72,7 +80,7 @@ class RegionPointCrawl {
for(def cityMap : cityList){
if(cityMap.level == "city"){
saveGaoDeCityRegionCode(province, cityMap.name, cityMap.citycode, cityMap.adcode, cityMap.center, cityMap.level);
saveGaoDeRegionCode(province, cityMap.districts);
saveGaoDeRegionCode(province, cityMap.name, cityMap.districts);
}
}
}
@@ -81,10 +89,10 @@ class RegionPointCrawl {
}
}
def saveGaoDeRegionCode(province, districts){
def saveGaoDeRegionCode(province, cityName,districts){
if(districts){
for(def dist : districts){
if(dist.level=="district"){
if(dist.level=="district" || cityName=="东莞市"){
saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level);
}
}
@@ -221,4 +229,3 @@ class RegionPointCrawl {
return KeyList.get(count);
}
}
H
@@ -6,6 +6,7 @@ import com.fangjia.core.util.HttpRequest
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
@@ -47,14 +48,13 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-06-01"
static def statDate = "2021-07-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
task.esfParser();//友达二手房存入mongo
// task.esfParser();//友达二手房存入mongo
// task.newhouseToSaveEs();//新房保存到es
task.esfhouseToSaveEs();//二手房保存到es
// task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间
}
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年7月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年8月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -308,10 +308,7 @@ class YouDaDataParser {
}
);
}
latch.await()
}
def saveEsfHouseToDB(def data){
@@ -367,15 +364,16 @@ class YouDaDataParser {
row.ydId = data[2].toInteger()
row.fjCity = city
def map = matchDistrict(city, row.region, row.districtName)
if(!map){
map = matchDistrict(city, row.region, row.address)
}
// def map = matchDistrict(city, row.region, row.districtName)
// if(!map){
// map = matchDistrict(city, row.region, row.address)
// }
def map = SeaweedDataMatchProcess.doMatch(city, row.region, row.districtName, row.address);
if(map){
row.fjRegion = map.region
row.fjName = map.name
row.lat = map.lat
row.lng = map.lng
row.lat = map.lat.toDouble()
row.lng = map.lng.toDouble()
row.location = map.lat.toString()+","+map.lng.toString()
def obj = CommonDatautils.findDistrictById(map.id.toString());
@@ -12,8 +12,10 @@ class FileUtils {
OutputStream out = new FileOutputStream(fileName, isAppend);
def sb = new StringBuffer();
sb.append(headers.join("|@|"))
sb.append(System.getProperty("line.separator"));
if(headers){
sb.append(headers.join("|@|"))
sb.append(System.getProperty("line.separator"));
}
out.write(sb.toString().getBytes(charset));
sb.setLength(0)
@@ -0,0 +1,465 @@
package com.fangjia.script.gj.data
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import com.rabbitmq.client.AMQP
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class AddDistrictFieldValue {
def detailColl = CrawalDBUtil.getDistrictDetailColl();
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40)
static main(args) {
def task = new AddDistrictFieldValue();
task.run();
}
def run(){
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
// for(def city : citys){
// def rets = findAllDataByCity(city)
// match(rets)
// }
//第三方均价数据填补
// for(def city : citys){
// def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat"])
// def rMap = getMatchDistrict(city)
// createResultValue(dists, rMap)
// }
//
for(def city : citys){
def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","name_cat","open_date","build_date","live_date", "build_type"])
// addFinishDate(dists)
addFieldValueByHouse(dists)
}
}
def match(def rets){
def totalCount = new AtomicLong()
def total = rets.size()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<rets.size();i++){
final def d = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def region = RegionData.formatRegion(d.city, d.region)
if(region) {
def rMap = SeaweedDataMatchProcess.doMatch(d.city, region, d.name, d.address);
if(rMap){
def q = new BasicDBObject()
q.put("_id", d._id)
def udata = new BasicDBObject()
udata.put("match_city", rMap.city)
udata.put("match_region", rMap.region)
udata.put("match_name", rMap.name)
detailColl.update(q, new BasicDBObject('$set',udata))
}
println totalCount.get()+"/${total}"+" "+d.city +" "+d.region+" "+d.name +" MATCH:"+rMap
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def findAllDataByCity(def city){
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
keys.put("name", 1)
keys.put("address", 1)
keys.put("match_name", 1)
def dbcursor = detailColl.find(new BasicDBObject("city", city), keys).batchSize(2000)
def rets = []
for(def d : dbcursor){
if(!d.match_name){
rets << d
}
}
return rets;
}
def getMatchDistrict(def city){
def q = new BasicDBObject()
q.put("city", city)
q.put("match_name", new BasicDBObject('$exists', true))
def dbcursor = detailColl.find(q)
def rMap = [:]
for(def d : dbcursor){
def key = d.match_city +"#"+d.match_region+"#"+d.match_name
def sourceMap = rMap.get(key)
if(!sourceMap){
sourceMap = [:]
rMap.put(key, sourceMap)
}
def value = sourceMap.get(d.root)
if(!value){
sourceMap.put(d.root, d)
}
}
return rMap;
}
def addFinishDate(dists){
def updateIndex = 0
def total = dists.size()
for(def d : dists){
if(!d.finish_building_date){
def udata = new BasicDBObject()
def newValue = FiledStrand.processFinishBuildingDate(d.live_date)
if(!newValue && d.open_date){
def date = d.open_date
newValue = date.getYear()+1900
}
if(!newValue && d.build_date){
newValue = FiledStrand.processFinishBuildingDate(d.build_date)
}
if(newValue){
udata.put("finish_building_date", newValue)
}
if(udata.size()>0){
updateIndex++
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
def createResultValue(dists, rMap){
def updateIndex = 0
def total = dists.size()
for(def d : dists){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
if(sourceMap){
def udata = new BasicDBObject()
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
udata.put("all_area", lastValue)
}
}
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
udata.put("build_area", lastValue)
}
}
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
udata.put("virescence_percent", lastValue)
}
}
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
udata.put("cubage_percent", lastValue)
}
}
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
udata.put("finish_building_date", lastValue)
}
}
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
udata.put("household_count", lastValue)
}
}
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
udata.put("car_site_count", lastValue)
}
}
if(udata.size()>0){
updateIndex++
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}
}
}
def getValuesByField(sourceMap, field, estateType){
def map = [:]
sourceMap.each{k,v->
def oldValue = v.get(field);
if(oldValue){
def newValue = null;
if(["address"].contains(field)){
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
}
if(["all_area","build_area"].contains(field)){
newValue = FiledStrand.processArea(oldValue)
}
if(["build_date"].contains(field)){
newValue = FiledStrand.processFinishBuildingDate(oldValue)
}
if(["house_count"].contains(field)){
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
}
if(["greening_rate"].contains(field)){
newValue = FiledStrand.processVirescencePercent(oldValue)
}
if(["volumetric_rate"].contains(field)){
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
}
if(["car_site_desc","car_site"].contains(field)){
newValue = FiledStrand.processCarSite(oldValue)
}
if(newValue){
def count = map.get(newValue)
if(!count){
map.put(newValue, 1)
}else{
map.put(newValue, count+1)
}
v.put(field, newValue)
}
}
}
def lastValue = null
if(map && map.size()>0){
map = map.sort {a, b -> b.value <=> a.value}
def values = map.values()
if(values.size()==1){
lastValue = map.keySet()[0]
}else{
def count1 = map.values()[0]
def count2 = map.values()[1]
if(count1 > count2){
lastValue = map.keySet()[0]
}else{
def rets = [map.keySet()[0],map.keySet()[1]]
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
for(def root : roots){
def data = sourceMap.get(root)
if(data){
def value = data.get(field)
if(rets.contains(value)){
lastValue = value
break
}
}
}
}
}
}
return lastValue;
}
def addFieldValueByHouse(def dists){
def updateIndex = new AtomicLong()
def totalCount = new AtomicLong()
def total = dists.size()
final CountDownLatch latch = new CountDownLatch(total)
for(def i=0;i<dists.size();i++){
final def d = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
if(d.estate_type2!="别墅" && (!d.finish_building_date || !d.build_type)){
def udata = new BasicDBObject()
def retMap = findHouses(d.city, d.region, "sell", d.name, d.alias)
if(retMap){
println retMap
if(!d.finish_building_date && retMap.containsKey("build_date")){
def dateMap = retMap.get("build_date")
if(dateMap){
def value = dateMap.keySet()[0]
if(value){
udata.put("finish_building_date", value)
}
}
}
if(!d.build_type && retMap.containsKey("height")){
def heightMap = retMap.get("height")
if(heightMap){
def heights = heightMap.keySet()
if(heights){
def max = heights.max()
if(max<=6){
udata.put("build_type", "多层")
}else if(max>6 && max<=11){
udata.put("build_type", "小高层")
}else if(max>11 && max<30){
udata.put("build_type", "高层")
}else if(max>30){
udata.put("build_type", "超高层")
}
}
}
}
}
if(udata.size()>0){
updateIndex.addAndGet(1)
def q = new BasicDBObject()
q.put("_id", d._id)
println updateIndex.get() +"/${total} "+d.city +" "+d.name+" "+udata
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
}
}else{
println totalCount.get() +" "+d.city +" "+d.name
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
latch.await(5, TimeUnit.MINUTES);
}
def findHouses(def city, def region, def cat, def name, def alias){
// QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region))
// must(QueryBuilders.termQuery("cat", cat)).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def restMap = [:]
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-*", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
DBObject seawater = new BasicDBObject(map)
if(seawater.build_date){
def newValue = FiledStrand.processFinishBuildingDate(seawater.build_date)
if(newValue){
def valueMap = restMap.get("build_date")
if(!valueMap){
valueMap = [:]
restMap.put("build_date", valueMap)
}
def count = valueMap.get(newValue)
if(!count){
valueMap.put(newValue, 1)
}else{
valueMap.put(newValue, count + 1)
}
}
}
if(seawater.height){
def hightMap = restMap.get("height")
if(!hightMap){
hightMap = [:]
restMap.put("height", hightMap)
}
def count = hightMap.get(seawater.height)
if(!count){
count = 1
hightMap.put(seawater.height, count)
}else{
hightMap.put(seawater.height, count + 1)
}
}
}
}
if(restMap){
def dateMap = restMap.get("build_date")
if(dateMap){
dateMap = dateMap.sort {a, b -> b.value <=> a.value}
restMap.put("build_date", dateMap)
}
}
return restMap;
}
}
@@ -15,7 +15,19 @@ class BestDistrictPriceExport {
static main(args) {
def task = new BestDistrictPriceExport();
task.run()
//task.run()
task.run2()
}
def run2(){
def filePath = "d:\\数据\\广发\\楼盘.xlsx"
ExcelUtils.multiThreadingModify(filePath, { row ->
def stat = findDistrictPrice(row['A'], row['B'], row['D'], "20210801")
if(stat){
row['S'] = stat.avg_price
}
println row['A']+" "+row['B']+" "+ row['C']+" "+row['S']
})
}
def run(){
@@ -24,10 +24,10 @@ class DataProcess {
static main(args) {
//建行信用卡城市(临汾--晋城).xlsx
def filePath = "D:\\数据\\建行银行\\建行信用卡入库小区V1.xlsx"
def filePath = "D:\\数据\\小区\\地图\\兰州.xlsx"
def task = new DataProcess();
// task.batchSaveDataToDB(filePath)
// task.mapBatchSaveDataToDB(filePath)
task.mapBatchSaveDataToDB(filePath)
// task.calcutePrice(filePath);
// task.calcutePrice2();
// task.deleteRepDist("株洲");
@@ -35,7 +35,7 @@ class DataProcess {
// task.standartName(filePath);
task.saveGaoDeMap(filePath);
// task.saveGaoDeMap(filePath);
// task.updateDBAddress();
// task.updateRegion()
@@ -120,13 +120,10 @@ class DataProcess {
def lastDatas = []
if(datas){
def keys = []
for(def data : datas){
if(data['A']=="厦门" && data['I'] && data['I'].toString().startsWith("24")){
for(def data : datas){//20,21,25,
if(data['I'] && [20,21,26,27].contains(data['I'].toString().toDouble().toInteger())){
def key = data['A']+"##"+data['B']+"##"+data['H']
if(!keys.contains(key) && data['H'].size()>2){
// if(data['I'].toInteger()==22){//&& data['N']<3
// continue;
// }
if(!keys.contains(key) && data['H'] && data['H'].size()>2){
keys << key
lastDatas << data
}
@@ -168,7 +165,7 @@ class DataProcess {
addIndex++
def values = saveData(d['A'], d['B'], d['H'], null, addr, estateType, d['F'], d['G'], null,null)
println "${addIndex}【新增入库】"+ values
// FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['B'], d['H'], 4);
}else{
println "【已存在】 ${d['A']} ${d['B']} ${d['H']}"
}
@@ -5,6 +5,7 @@ import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
@@ -26,8 +27,9 @@ class ExportData {
static main(args) {
def task = new ExportData();
// task.exportThird()
task.exportMap();
task.exportThird()
// task.exportMap();
}
def exportMap(){
@@ -72,7 +74,7 @@ class ExportData {
def exportThird(){
def citys = ["ÖØÇì"]
def citys = ["Äϲı"]
for(def city : citys){
def rets = []
rets << ["城市","区域","名称","别名"]
@@ -1,47 +1,65 @@
package com.fangjia.script.gj.district
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.script.yinjihaun.lng.UpdateDistrictLocation
import com.mongodb.BasicDBObject
import java.util.regex.Pattern
/***
* 地图小区处理
*/
class MapDistrictProcess {
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
static main(args) {
def task = new MapDistrictProcess();
task.run()
// task.updateRegion()
}
/**
* 1:删除 10:小区已存在 11:去关键字后相同删除 12:距离排除 13:人工删除 14:得分小于0.6 15:预测写字楼 16:人工删除百度房地产类型 17:词性删除 18:名称字数太少小于3 19:路
* 20:不同区域新增(库中无) 21:分期新增 22:分数在0.6以上
* 20:不同区域新增(库中无) 21:分期新增 23:不同区域,距离相近 25:无匹配,可新增 22:有匹配,距离1000米以下,相似度在0.6以上 24:疑似匹配有误 27:人工判定入库
* @return
*/
def run(){
def files = ["武汉.xlsx"]
// def files = ["重庆.xlsx","东莞.xlsx","温州.xlsx","绍兴.xlsx","宁波.xlsx","南昌.xlsx","济南.xlsx","金华.xlsx","肇庆.xlsx","湛江.xlsx","岳阳.xlsx","盐城.xlsx","唐山.xlsx","三亚.xlsx","廊坊.xlsx","呼和浩特.xlsx","西宁.xlsx","日照.xlsx","常德.xlsx"]
//"乌鲁木齐.xlsx","烟台.xlsx","保定.xlsx","贵阳.xlsx","湖州.xlsx","惠州.xlsx","吉林.xlsx","遵义.xlsx","宜昌.xlsx","芜湖.xlsx","兰州.xlsx"
def files = ["兰州.xlsx"]
for(def fileName : files){
def filePath = "D:\\数据\\小区\\地图\\"+fileName;
// typeFilter(filePath)
// nameStandard(filePath);
// dataFilter(filePath)
matchDistDatas(filePath)
typeFilter(filePath)
nameStandard(filePath);
dataFilter(filePath)
houseCalculate(filePath);
////// //统计是否存在多个区域
//统计是否存在多个区域
matchDistDatas(filePath)
moreRegionCalculate(filePath)
//
addFlag(filePath)
mapMatch(filePath);
baiduTypeFilter(filePath);
baiduMap(filePath);
// baiduMap(filePath);
distanceFilter(filePath)
predictType(filePath)
noMatchFlag(filePath)
matchFlag(filePath)
addFeature(filePath)
houseMatch(filePath);
predictType(filePath)
// nameScore(filePath);
nameScore(filePath);
thirdPartyData(filePath)
}
}
@@ -55,7 +73,12 @@ class MapDistrictProcess {
row['I'] = 1
}else{
if(name.toString().length()<3){
row['I'] = 18
if(row['N'] && row['N']>=10 && !row['J']){
row['I'] = 25
}else{
row['I'] = 18
}
}else{
def s1 = DataUtil.match(name, Pattern.compile("(路|街|道)\\d+号\$"))
if(s1){
@@ -70,7 +93,7 @@ class MapDistrictProcess {
}
});
println "2.名称过滤完成"
println "3.名称过滤完成"
}
def typeFilter(filePath){
@@ -84,15 +107,30 @@ class MapDistrictProcess {
println "1.类型过滤完成"
}
def baiduTypeFilter(filePath){
def types = ["房地产;住宅区","房地产;内部楼栋","房地产","房地产;宿舍"]
ExcelUtils.multiThreadingModify(filePath, { row ->
def type = row['S']
if(type && !types.contains(type)){
if(type && row['Q'] && row['Q']==row['H'] && !types.contains(type)){
row['I'] = 1
}
});
println "1.类型过滤完成"
println "9.百度类型过滤完成"
}
def distanceFilter(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']){
if(row['X'] && row['Y']){
if(row['X']<=500 && row['Y']>=0.5){
row['I'] = 12
}
}
}
});
println "10.距离过滤完成"
}
def nameStandard(filePath){
@@ -105,12 +143,12 @@ class MapDistrictProcess {
}
}, 0);
println "1.名称标准化完成"
println "2.名称标准化完成"
}
def matchDistDatas(def path){
def index = 0
ExcelUtils.modify(path,{ List row ->
ExcelUtils.multiThreadingModify(path,{ List row ->
index ++
if(!row['I'] && !row['J']){
def rMap = SeaweedDataMatchProcess.doMatch(row['A'], row['B'], row['H'], row['D']);
@@ -126,13 +164,16 @@ class MapDistrictProcess {
def maplat = baidu.y
row['X'] = GeoHelper.GetDistance(maplat, maplng, rMap.lat, rMap.lng)
def score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['L'])
if(score<0.3){
score = CosineSimilarAlgorithm.getSimilarity(row['H'], row['M'])
}
row['Y'] = score
}
}
println index +" "+row
}
},0)
println "3.小区匹配完成"
println "5.小区匹配完成"
}
def houseCalculate(filePath){
@@ -141,6 +182,7 @@ class MapDistrictProcess {
row['N'] = EsSearchUtils.findHouse(row['A'], row['B'], row['H'])
}
});
println "4.房源统计完成"
}
def moreRegionCalculate(filePath){
@@ -161,8 +203,12 @@ class MapDistrictProcess {
}
}
if(!regions.contains(row['B'])){
row['O'] = regions.toString()
row['I'] = 20
if(row['X'] && row['X']<=500 && row['Y'] && row['Y']>=0.5){
row['I'] = 23
}else{
row['O'] = regions.toString()
row['I'] = 20
}
}else if(regions.contains(row['B'])){
row['O'] = "已存在"
row['I'] = 10
@@ -170,11 +216,12 @@ class MapDistrictProcess {
}
}
});
println "6.区域过滤完成"
}
def addFlag(def filePath){
def index = 0
ExcelUtils.modify(filePath, { row ->
ExcelUtils.multiThreadingModify(filePath, { row ->
index++
if(!row['I']) {
//根据别名查找
@@ -206,6 +253,8 @@ class MapDistrictProcess {
println index + " " + row
}
});
println "7.打标记完成"
}
def baiduMap(def filePath){
@@ -233,6 +282,7 @@ class MapDistrictProcess {
}
}
});
println "11.特征过滤完成"
}
def mapMatch(def filePath){
@@ -260,16 +310,22 @@ class MapDistrictProcess {
println index +" "+row
},0);
println "4.地图匹配完成"
println "8.地图匹配完成"
}
def houseMatch(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']) {
def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
if(rMap){
row['I'] = rMap.flag
}
// def rMap = FlagProcess.addDataFlagByHouseCount(row['J'], row['K'], row['L'], row['G'], row['F'], row['N'])
// if(rMap){
// row['I'] = rMap.flag
// }
if(row['J']){
if(row['X'] && row['Y'] ){
}
}
}
},0);
}
@@ -314,6 +370,41 @@ class MapDistrictProcess {
},0);
}
def noMatchFlag(filePath) {
ExcelUtils.multiThreadingModify(filePath, { row ->
if (!row['I'] && !row['J']) {
if(row['N']>0){
row['I'] = 25
}else{
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
row['I'] = 25
}
}
}
});
}
def matchFlag(filePath) {
ExcelUtils.multiThreadingModify(filePath, { row ->
if (!row['I'] && row['J'] && row['X'] && row['Y']) {
if(row['X']<=1000 && row['Y']>=0.6){
row['I'] = 22
}else if(row['X']<=600 && row['Y']>=0.5){
row['I'] = 22
}else if(row['X']<=1300 && row['Y']>=0.8){
row['I'] = 22
}else{
row['I'] = 24
}
}else if(!row['I'] && row['J'] && row['Y']==0){
if(row['E']!='商务住宅;商务住宅相关;商务住宅相关'){
row['I'] = 26
}
}
});
}
def nameScore(filePath){
ExcelUtils.multiThreadingModify(filePath, { row ->
if(!row['I']) {
@@ -343,4 +434,59 @@ class MapDistrictProcess {
}
},0);
}
def updateRegion(){
def filePath = "D:\\数据\\小区\\地图\\东莞.xlsx";
ExcelUtils.multiThreadingModify(filePath, { row ->
def region = getRegionByLocation(row['F'], row['G']);
if(region){
row['B'] = RegionData.formatRegion(row['A'], region)
}
},0)
}
def thirdPartyData(filePath){
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
ExcelUtils.multiThreadingModify(filePath, { row ->
if(row['I'] && [20,21,25,26,27].contains(row['I'].toString().toDouble().toInteger())){
def rets = findIsExistsOtherWebsite(row['A'], row['B'], row['H'])
if(rets){
row['W'] = rets.join(",")
}
}
},0)
}
def findIsExistsOtherWebsite(def city, def region, def name){
def q = new BasicDBObject()
q.put("city",city)
q.put("region",region)
q.put("name", name)
def dbcursor = districtDetailColl.find(q)
def rets = []
for(def r : dbcursor){
if(!rets.contains(r.root)){
rets << r.root
}
}
return rets
}
def getRegionByLocation(lng, lat){
def jsonObject = GaoDeApiUtils.geocode(lng.toString()+","+lat.toString())
if(jsonObject && jsonObject.info=="OK"){
def compent = jsonObject.regeocode.addressComponent
if(compent){
def township = compent.township
return township;
}
}
return null
}
}
@@ -7,7 +7,9 @@ import java.util.regex.Pattern
class NameProcess {
static def filterName(def name){
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心","养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会"]
def keys = ["散盘","办公楼","写字楼","商务","家居建材","商贸城","服装城","科技园","删除","五金","购物广场","物流园","文化教育","材料城","充电站","产业园","商业中心","办公大楼","工业","机械城","孵化","体验中心","投诉站","接送点","文化站","创业","办公中心","自编","创新","科技","实业园","管理处","百货","展园","展厅","软件园","基地","学生","青年","女生","学生宿舍","单身","办公室","值班室","服务站","酒店宿舍","男生","地块","租房","临时","大门","农业园","装修","摆件","展示","发展中心","商业广场","运营中心","店铺","筹备组","车场","厂家","培训","号门","管理科","招商处","展点","住院楼","家禽","捐赠","群租","招租","软装","生活广场","直购","员工","理赔","互助","赠品","图片","男宿舍","招租","站点","卸货","购物","大学","消火栓","集散地","过渡房","多肉","等候","租","商铺","家庭式","地产","宾馆","共享","人才","创意","轰趴","会所","不动产","美丽","精装","专区","下客点","办公","设计","汽车","垃圾","圆床房","白领","厂房","水果","专家","的","人员","自如","影吧","物业","会展","文明楼","工房","示范","宣传","老年","妇女","沐浴","蔬果","敬老","棋牌","房产","公益","碧波庭","高层","服务","居民","英才","职工公寓","职工之家","风情园","中心","项目","门卫","客栈","底商","美团","空间","无人机","国际广场","智慧社区","食品","排","交口","爱心",
"养老","集贸","保安","建筑工人","单位宿舍","工程","演员","二人转","建工","兵营","供应","土产","学员","职工","门口","欢迎","车棚","宝宝","陵园","登记处","植物","驿站","自助","书","女子","健康","营业","扎啤","创客","电子广场","村民安置房","办事处生活区","萌宠","啤酒","高档","回迁楼","品茶","公司宿舍","置业","明星","家庭","老区","新区","单元","居民点","救助站","公司宿舍","梦想","公司职工宿舍","公司职工住宅楼","单位房","平安","文明社区","书院","丝绸","旧区","建材城","栋宿舍","吉宅","女人城","疾控","医院","易初莲花","发展","魔方","轻纺城","合作","外贸","嬉空间","优+","医生","康乃馨","总部","网格站","私人定制","家装","换新家","仓库","防盗","客运站","蔬菜","旁","爱情","菜场","套","商场","公寓房","守望点","硕士","佛庵","组团","区宿舍","妇幼","英才","血站","女儿","环卫","贸易城","平房","公安","物流","管理","理想","商业城","留医部","劳动","医药","团校","聚贤","豪华","电梯","超豪","恋家","地铁","法院","高级公寓","温馨","安全文明","精英","教师村","旧村改造","私宅","楼群","老师楼","敬业","佳缘","保障性住房","自建房","群众","度假","政府住宅","乘务","科教","房改房","豪宅","移民","艺校","运管处","校区","教委楼","学区","教师公寓","记者站","温泉","旅店","防疫站","园区","扶贫","按摩","文化中心","居委会", "限价房","看房",
"合伙人","中学公寓","海景房","儿童","花艺","乐乐家","招收","商业","研究生","教师","旅游","精洗","物管区","律师","果蔬","文化室","休闲广场","集资房","精神家园","红星美凯龙","雅居","高中","球室","世华易居","住院综合楼","养生"]
def isAdd = true;
for(def key : keys){
if(name.contains(key)){
@@ -17,7 +19,7 @@ class NameProcess {
}
if(isAdd){
def endKeys = ["路","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍"]
def endKeys = ["路","街","店","酒店","汽车城","五金城","机电城","装饰城","材料城","农贸","商城","工业区","家居广场","家居生活中心","服务中心","创新中心","科创中心","财智中心","金融中心","创投中心","外包中心","创业大厦","度假","商写楼","塑化城","材料城","礼品城","汽车用品城","汽配城","市场","批发","文创园","交易中心","服装中心","科技中心","科创园","工业园","创业园","度假区","科创谷","旅游城","食品城","工业坊","百货","商贸广场","采购中心","购物中心","小学","教学楼","学区房","有限公司","电力站","模具城","装饰","数码广场","教师楼","茶城","建材家居","物流城","器材广场","贸易中心","商品城","建材广场","商贸街","绣品街","学生","高层","养生谷","度假项目","工程","度假村","度假区","早市","少儿","药铺","温馨","单间","公司住宅","公司宿舍","校宿舍","格格","茶叶","人行门","学校","公司"]
for(def key : endKeys){
if(name.toString().endsWith(key)){
isAdd = false;
@@ -27,7 +29,7 @@ class NameProcess {
}
if(isAdd){
def startKeys = ["FQ"]
def startKeys = ["FQ","社区"]
for(def key : startKeys){
if(name.toString().endsWith(key)||name.toString().startsWith(key)){
isAdd = false;
@@ -36,7 +38,7 @@ class NameProcess {
}
}
if(isAdd){
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房"]
def strs = ["碧桂园","雅居乐","满庭芳","综合商住楼","教育宿舍","生活小区","市政院","公交小区","搬迁小区","家属楼","商业小区","委宿舍","人行宿舍","工人宿舍","教师公寓","教师宿舍","顺心居","小花园","医药宿舍","商住小区","统建楼","农民","供销楼","温馨楼","星期","党校","统建楼","惜缘","改造","国际城","好客来","商业大楼","一二期","市政楼","非小区","高中","建委楼","商业社区","双建委","教育小区","饭店","建安处","开发区","生活区","和社区","家属院","经济适用房", "家属宿舍楼","卫生生活区"]
for(def str : strs){
if(str == name){
isAdd = false;
@@ -56,7 +58,7 @@ class NameProcess {
}
}
//.replace("院小区","小区")replace("号小区","号")..replace("号院","号").replace("号住宅区","号")
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").trim()
name = name.replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("【","").replace("】","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期").replace(" ","").replace("Ⅰ","").trim()
name = name.replace("(建设中)","").replace("售楼中心","").replace("销售中心","").replace("营销中心","").replace("接待处","").replace("售楼部","").replace("房产销售","").replace("售楼处","").replace("销售部","").replace("商品房","")
if(name && name.contains("区")){
def s0 = DataUtil.match(name, Pattern.compile("[a-zA-Z]\\d+区"))
@@ -191,12 +193,12 @@ class NameProcess {
if(str == name){
return null
}
if(name){
s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
if(s1){
return null
}
}
// if(name){
// s1 = DataUtil.match(name, Pattern.compile("^\\w+"))
// if(s1){
// return null
// }
// }
return name.toUpperCase()
}
@@ -15,13 +15,19 @@ class SeaweedDataMatchProcess {
if(!rMap.city){
def obj = CommonDatautils.findDistrict(city, region, name, "district")
if(!obj){
obj = CommonDatautils.findDistrictByAlias(city, region, name, "district")
def datas = CommonDatautils.findDistrictByAlias(city, region, name, "district")
if(datas){
obj = datas[0]
}
}
if(obj){
rMap.estate_type2 = obj.estate_type2
rMap.city = obj.city
rMap.region = obj.region
rMap.name = obj.name
rMap.lat = obj.lat/10000000000.0
rMap.lng = obj.lng/10000000000.0
rMap.id = obj._id.toString()
}
}
}
@@ -40,14 +46,14 @@ class SeaweedDataMatchProcess {
if(rMap && rMap.id && region.contains(rMap.region)) {
def obj = CommonDatautils.findDistrictById(rMap.id.toString());
if (obj) {
rMap.estate_type2 = obj.estate_type2
}
}else{
rMap = null
}
// if(rMap && rMap.id && region.contains(rMap.region)) {
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
// if (obj) {
// rMap.estate_type2 = obj.estate_type2
// }
// }else{
// rMap = null
// }
if(!rMap && name && name.contains("¡¤")){
def newName = trimChar(name).replace("¡¤","")
rMap = matchFromDB2(city,newName)
@@ -0,0 +1,320 @@
package com.fangjia.script.gj.districtwash
import com.fangjia.core.RegionData
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.FiledStrand
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
class AddDetailDistrict {
def detailColl = CrawalDBUtil.getDistrictDetailColl();
static main(args) {
def task = new AddDetailDistrict();
// task.run();
// task.delRepeatDist
task.addEstateType()
}
def addEstateType(){
def citys = ["沈阳","哈尔滨","吉林","遵义","淄博","西宁","青岛","大连","宁波"]
for(def city : citys){
def keys = []
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","estate_type2"])
for(def d : districts) {
if(!d.estate_type2){
def estateType2 = "普通住宅"
if(d.name.contains("别墅")){
estateType2 = "别墅"
}else if(d.name.contains("商住")){
estateType2 = "商住"
}
DBObject update = new BasicDBObject();
update.put('$set', new BasicDBObject("estate_type2", estateType2).append("m_date", new Date()));
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
}
}
println city
}
}
def delRepeatDist(){
def citys = CcbCommonUtils.citys
for(def city : citys){
def keys = []
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
def index = 0
for(def d : districts){
index++
def key = d.city +"##"+d.region+"##"+d.name
if(!keys.contains(key)){
keys << key
}else{
println index +" "+d.region+" "+d.name
DBObject update = new BasicDBObject();
update.put('$set', new BasicDBObject("status", 1).append("m_date", new Date()));
DbUtils.seaweed.update(new BasicDBObject("_id", d.get("_id")), update)
}
}
}
}
def run() {
def citys = ["沈阳", "哈尔滨", "吉林", "遵义", "淄博", "西宁", "青岛", "大连", "宁波"]//CcbCommonUtils.citys;
//第三方均价数据匹配
for (def city : citys) {
def rets = findAllDataByCity(city)
def rMap = getMatchDistrict(city)
toDB(rets, rMap)
}
}
def toDB(def rets, def rMap){
def saveDistKeys = []
def total = rets.size()
def index = 0
def addIndex = 0
for(def d : rets){
index++
def newRegion = RegionData.formatRegion(d.city, d.region)
def name = NameProcess.nameStandard(d.city, newRegion, d.name)
if(!name){
continue
}
def flag = NameProcess.filterName(name)
def key = null
if(newRegion){
key = d.city+"#"+newRegion+"#"+name
}
if(flag && !saveDistKeys.contains(key)){
def gdData = FangjiaClientUtils.getMapData(d.city, newRegion, name, d.address,"", "高德")
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(gdData){
if(!newRegion){
newRegion = RegionData.formatRegion(d.city, gdData.district)
if(newRegion){
key = d.city+"#"+newRegion+"#"+name
}
}
}
if(key && gdData && gdData.lng){
def matchMap = SeaweedDataMatchProcess.doMatch(d.city, newRegion, name, d.address);
if(!matchMap){
addIndex++
if(gdData.lng){
def gmap = GeocodeUtils.getGaodeToBaidu(gdData.lng.toDouble(), gdData.lat.toDouble())
lng2 = (gmap.get("lng").toDouble()* 10000000000).longValue()
lat2 = (gmap.get("lat").toDouble()* 10000000000).longValue()
lng = (gdData.lng.toDouble()* 10000000000).longValue()
lat = (gdData.lat.toDouble()* 10000000000).longValue()
}
def newAddress = DataUtil.formateAddress(d.city, d.address)
def udata = createFieldValue(d, rMap)
udata.put("lng", lng)
udata.put("lat", lat)
println index+"/${addIndex}/${total} "+d.city+" "+newRegion+" "+name+" "+d.root+" "+udata
udata.put("city", d.city)
udata.put("region", newRegion)
udata.put("name", name)
udata.put("address", newAddress)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("status", 0)
udata.put("cat", "district")
udata.put("c_date", new Date())
udata.put("m_date", new Date())
DbUtils.seaweed.save(udata)
saveDistKeys << key
}
}
// println d.city+" "+ d.region+" "+d.name
}
}
}
def createFieldValue(d, rMap){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
def udata = new BasicDBObject()
if(sourceMap){
if(!d.address){
def lastValue = getValuesByField(sourceMap,"address", d.estate_type2)
if(lastValue){
udata.put("address", lastValue)
}
}
if(!d.all_area){
def lastValue = getValuesByField(sourceMap,"all_area", d.estate_type2)
if(lastValue){
udata.put("all_area", lastValue)
}
}
if(!d.build_area){
def lastValue = getValuesByField(sourceMap,"build_area", d.estate_type2)
if(lastValue){
udata.put("build_area", lastValue)
}
}
if(!d.virescence_percent){
def lastValue = getValuesByField(sourceMap,"greening_rate", d.estate_type2)
if(lastValue){
udata.put("virescence_percent", lastValue)
}
}
if(!d.cubage_percent){
def lastValue = getValuesByField(sourceMap,"volumetric_rate", d.estate_type2)
if(lastValue){
udata.put("cubage_percent", lastValue)
}
}
if(!d.finish_building_date){
def lastValue = getValuesByField(sourceMap,"build_date", d.estate_type2)
if(lastValue){
udata.put("finish_building_date", lastValue)
}
}
if(!d.household_count){
def lastValue = getValuesByField(sourceMap,"house_count", d.estate_type2)
if(lastValue){
udata.put("household_count", lastValue)
}
}
if(!d.car_site_count){
def lastValue = getValuesByField(sourceMap,"car_site", d.estate_type2)
if(lastValue){
udata.put("car_site_count", lastValue)
}
}
}
return udata
}
def getValuesByField(sourceMap, field, estateType){
def map = [:]
sourceMap.each{k,v->
def oldValue = v.get(field);
if(oldValue){
def newValue = null;
if(["address"].contains(field)){
newValue = DataUtil.formateAddress(v.get("city"), oldValue)
}
if(["all_area","build_area"].contains(field)){
newValue = FiledStrand.processArea(oldValue)
}
if(["build_date"].contains(field)){
newValue = FiledStrand.processFinishBuildingDate(oldValue)
}
if(["house_count"].contains(field)){
newValue = FiledStrand.processHouseholdCount(oldValue, estateType)
}
if(["greening_rate"].contains(field)){
newValue = FiledStrand.processVirescencePercent(oldValue)
}
if(["volumetric_rate"].contains(field)){
newValue = FiledStrand.processCubagePercent(oldValue, estateType)
}
if(["car_site_desc","car_site"].contains(field)){
newValue = FiledStrand.processCarSite(oldValue)
}
if(newValue){
def count = map.get(newValue)
if(!count){
map.put(newValue, 1)
}else{
map.put(newValue, count+1)
}
v.put(field, newValue)
}
}
}
def lastValue = null
if(map && map.size()>0){
map = map.sort {a, b -> b.value <=> a.value}
def values = map.values()
if(values.size()==1){
lastValue = map.keySet()[0]
}else{
def count1 = map.values()[0]
def count2 = map.values()[1]
if(count1 > count2){
lastValue = map.keySet()[0]
}else{
def rets = [map.keySet()[0],map.keySet()[1]]
def roots = ["lianjia.com", "ke.com","fang.com","58.com","anjuke.com","5i5j.com","jiwu.com","centanet.com","house365.com","zhuge.com"]
for(def root : roots){
def data = sourceMap.get(root)
if(data){
def value = data.get(field)
if(rets.contains(value)){
lastValue = value
break
}
}
}
}
}
}
return lastValue;
}
def getMatchDistrict(def city){
def q = new BasicDBObject()
q.put("city", city)
// q.put("match_name", new BasicDBObject('$exists', true))
def dbcursor = detailColl.find(q)
def rMap = [:]
for(def d : dbcursor){
def key = d.city +"#"+d.region+"#"+d.name
def sourceMap = rMap.get(key)
if(!sourceMap){
sourceMap = [:]
rMap.put(key, sourceMap)
}
def value = sourceMap.get(d.root)
if(!value){
sourceMap.put(d.root, d)
}
}
return rMap;
}
def findAllDataByCity(def city){
def q = new BasicDBObject("city", city);
q.put("match_name", new BasicDBObject('$exists', false))
q.put("avg_price", new BasicDBObject('$gt', 0))
def dbcursor = detailColl.find(q).batchSize(2000)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -1,8 +1,9 @@
package com.fangjia.script.gj
package com.fangjia.script.gj.districtwash
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.datastore.FieldValidatorUtils
import com.fangjia.script.gj.OpenApiUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.thirdPriceCrawl.Utils
import com.fangjia.script.util.DbUtils
@@ -0,0 +1,335 @@
package com.fangjia.script.gj.guangfa
import com.fangjia.core.RegionData
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.ccb.CcbCodeUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicLong
class GuangfaBlankSelectDistrict {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017";
public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl)
static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new GuangfaBlankSelectDistrict();
task.run()
}
def run(){
def file = "D:\\数据\\广发\\获取楼盘信息需求(发送版).xlsx"
def cityMap = standardRegion(file)
def priceMap = getZhiDaoPrice(file)
selectDistricts(cityMap, priceMap)
println "写入完成"
}
def selectDistricts(cityMap, priceMap){
def rets = []
rets << ["城市","区域","指导价","楼盘名称","楼盘地址","开发商名称","容积率","物业名称","*套数","*楼龄","户型","*建筑面积区间","2021年1-6月成交量","*均价","*成交量","*均价","*成交量","*均价"]
cityMap.each{city, regions->
if(["西安","东莞","杭州"].contains(city)){
def thirdPriceMap = findNearThirdPartyPrice(city, regions)
def retMaps = findCreditHightDistrict2(city, regions, "2021-08-01", priceMap, thirdPriceMap)
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias","estate_company", "developer","cubage_percent","household_count", "finish_building_date"])
//基本情况合格的小区
def distMap = [:]
for(def d : districts){
if(regions.contains(d.region)){// && d.household_count && d.household_count>=50
def distKeys = retMaps.get(d.region)
if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
if(d.finish_building_date){
def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
if(year){
def mod = 2021-year
if(mod <=15){
d.house_year = mod
}else{
d.house_year = null
}
}else{
d.house_year = null
}
}else{
d.house_year = null
}
distMap.put(d.city+"##"+d.region+"##"+d.name, d)
}
// if(d.finish_building_date){
// def year = CcbCodeUtils.parseBuildYear(d.finish_building_date)
// if(year){
// def mod = 2021-year
// if(mod <=15){
// def distKeys = retMaps.get(d.region)
// if(distKeys && distKeys.contains(d.city+"##"+d.region+"##"+d.name)){
// d.house_year = mod
// distMap.put(d.city+"##"+d.region+"##"+d.name, d)
// }
// }
// }
// }
}
}
println city +" 基本情况合格的小区:"+distMap.size()
for(def r : regions){
def names = retMaps.get(r)
if(names){
def results = getRoomTypeAndTradeInfo(names, distMap, priceMap)
rets.addAll(results)
}
}
println "结束"
}
}
println "开始写入Excel....."+rets.size()
ExcelUtils.write("d:\\数据\\广发\\楼盘(西安东莞杭州).xlsx", rets, "数据")
}
def getRoomTypeAndTradeInfo(def names, def distMap, def priceMap){
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(names.size())
for(def i=0;i<names.size();i++){
def key = names[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def d = distMap.get(key)
if(d){
def ret = getDetailInfo(d, priceMap)
if(ret){
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
// latch.await()
latch.await(5, TimeUnit.MINUTES);
return rets
}
def getDetailInfo(d, priceMap){
def retMap1 = null;
def retMap2 = null;
def retMap3 = null
if(d.city == "上海"){
retMap1 = findShTrades(d, "2021-01-01", "2021-07-01")
retMap2 = findShTrades(d, "2020-01-01", "2020-12-31")
retMap3 = findShTrades(d, "2019-01-01", "2019-12-31")
}else{
retMap1 = CcbCommonUtils.findTrades(d, "2021-01-01", "2021-07-01")
retMap2 = CcbCommonUtils.findTrades(d, "2020-01-01", "2020-12-31")
retMap3 = CcbCommonUtils.findTrades(d, "2019-01-01", "2019-12-31")
}
if(retMap1){
d.count1 = retMap1.count
d.avg_price1 = (int)retMap1.avg_price
}
if(retMap2){
d.count2 = retMap2.count
d.avg_price2 = (int)retMap2.avg_price
}
if(retMap3){
d.count3 = retMap3.count
d.avg_price3 = (int)retMap3.avg_price
}
if(!retMap1 || !retMap2 || !retMap3){
println d.city+" "+ d.region+" "+ d.name +"成交有一年为空过滤"
return null
}
def houses = EsSearchUtils.findSellHouseDetail(d.city, d.region, d.name)
def rooms = []
def areas = []
for(def h : houses){
rooms << h.room
areas << h.area
}
rooms = rooms.unique().sort()
areas = areas.unique().sort()
if(rooms){
d.room = rooms[0]+"-"+rooms[rooms.size()-1]
d.area = areas[0]+"-"+areas[areas.size()-1]
}
def zhidaoprice = priceMap.get(d.city+"#"+d.region)
def ret = [d.city, d.region, zhidaoprice, d.name, d.address, d.developer, d.cubage_percent, d.estate_company, d.household_count, d.house_year,d.room, d.area, d.count1, d.avg_price1,d.count2, d.avg_price2,d.count3, d.avg_price3]
println ret
return ret;
}
def standardRegion(filePath){
def city = null
def cityMap = new LinkedHashMap()
def index = 0
ExcelUtils.modify(filePath, { row ->
index ++
if(index >= 4){
if(row['C']){
city = row['C']
}
println city+" "+row['D']
def newRegion = null
if(city && row['D']){
newRegion = RegionData.formatRegion(city, row['D'])
row['F'] = newRegion
}
if(city && newRegion){
def regions = cityMap.get(city)
if(!regions){
regions = []
cityMap.put(city, regions)
}
if(!regions.contains(newRegion)){
regions << newRegion
}
}
}
});
return cityMap
}
def findCreditHightDistrict2(def city, def regions, def weekend, def priceMap, def thirdPriceMap){
def q = new BasicDBObject()
q.put("city", city)
q.put("value_type", 2)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
q.put("region", new BasicDBObject('$in', regions))
def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1))
def retMaps = [:]
for(def d : dbcursor){
if(regions.contains(d.region)){
def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name)
if(thirdPrice){
def precent = (thirdPrice-d.avg_price)*100/d.avg_price
if(Math.abs(precent)>=0 && Math.abs(precent)<=10){
def price = priceMap.get(d.city +"#"+d.region)
if(d.avg_price>price){
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
}else{
def rets = retMaps.get(d.region)
if(!rets){
rets = []
retMaps.put(d.region, rets)
}
rets << d.city+"##"+d.region+"##"+d.district_name
}
}
}
return retMaps;
}
def getZhiDaoPrice(filePath){
def priceMap = [:]
def city = null
ExcelUtils.modify(filePath, { row ->
if(row['C']){
city = row['C']
}
if(city && row['F']){
def key = city +"#"+row['F']
if(!priceMap.containsKey(key)){
priceMap.put(key, row['E'])
}
}
})
return priceMap
}
public static def findShTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("fjCity", dist.city)
q.put("fjRegion", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("fjName", new BasicDBObject('$in', dist.alias))
}else{
q.put("fjName", dist.name)
}
q.put("contractTime", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = youda_esfhouse_coll.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.price){
total_price = total_price + d.price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
if(count && avg_price){
def retMap = [:]
retMap.count = count
retMap.avg_price = avg_price
return retMap
}
return null
}
def findNearThirdPartyPrice(def city, regions){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", new BasicDBObject('$in', regions))
q.put("s_date", new BasicDBObject('$gte', 202105))
q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q)
def thirdPriceMap = [:]
for(def d : dbcursor){
def key = d.city+"#"+d.region+"#"+d.name
if(!thirdPriceMap.containsKey(key)){
thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price)
}
}
return thirdPriceMap
}
}
@@ -4,6 +4,7 @@ import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.looplineinfo.Point
@@ -14,7 +15,13 @@ import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern;
@@ -22,59 +29,81 @@ import java.util.regex.Pattern;
class LocationTest {
static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
Date date=new Date(1622802609);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String str = format.format(date);
System.out.println(str);
// Date date=new Date(1622802609);
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
// String str = format.format(date);
// System.out.println(str);
// def task = new LocationTest()
def task = new LocationTest()
// task.run()
// task.run2()
// task.run3()
task.run3()
// task.run4()
// task.run5()
}
def run(){
def regionPointsMap = findRegionRange()
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region", "name", "address", "lat", "lng"]);
def city = "杭州"
def regionPointsMap = findRegionRange(city)
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat", "lng"]);
def index = 0
def rets = []
for(def dist : districts){
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
if(!flag){
index++
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
for(def i=0;i<districts.size();i++){
final def dist = districts[i]
def loactionMap = matchMap(dist)
if(loactionMap){
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat/10000000000.0, dist.lng/10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if(dist.region==loactionMap.region){
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,与地图相同"
}else{
println index+" "+dist.region +" "+dist.name +" 不在区域范围内 地图:"+ loactionMap.region+" "+distance
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def flag = isInArea(dist.lng/10000000000.0, dist.lat/10000000000.0, regionPointsMap.get(dist.region))
if(!flag) {
index++
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
def loactionMap = matchMap(dist)
if (loactionMap) {
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if (dist.region == loactionMap.region) {
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
} else {
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
}
} else {
ret << ""
println totalCount.get() + " " + dist.region + " " + dist.name + " 不在区域范围内,地图没找到"
}
if (ret) {
rets << ret
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}else{
ret << ""
println index+" "+dist.region +" "+dist.name +" 不在区域范围内,地图没找到"
}
rets << ret
}
});
}
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误5.xlsx", rets, "区域不在行政范围内")
latch.await()
ExcelUtils.write("d:\\数据\\福州\\区域经纬度错误-${city}.xlsx", rets, "区域不在行政范围内")
}
def run2(){
@@ -133,10 +162,12 @@ class LocationTest {
def run3(){
ExcelUtils.modify("d:\\数据\\福州\\区域经纬度错误5.xlsx", {d->
def index = 0
ExcelUtils.modify("d:\\数据\\福州\\经纬度错误.xlsx", {d->
index++
def distance = null
if(d['G']){
distance = d['G'].toDouble()
if(d['H']){
distance = d['H'].toDouble()
}
if(d['E'] && d['B'] == d['E'] && distance && distance>=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
@@ -180,12 +211,59 @@ class LocationTest {
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
def dbo = DbUtils.seaweed.findOne(q)
d['K'] = "修正"
}
}else if(d['E'] && d['B'] != d['E'] && distance && distance<=1000){
def mapName = d['F'].toString().replace("(建设中)","").replace("?","").replace("&","").replace("·","").replace(".","").replace("(","").replace(")","").replace("(","").replace(")","").replace("11期","十一期").replace("12期","十二期").replace("13期","十三期").replace("14期","十四期").replace("15期","十五期").replace("16期","十六期").replace("17期","十七期").replace("18期","十八期").replace("19期","十九期").replace("10期","十期").replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","六期").replace("7期","七期").replace("8期","八期").replace("9期","九期")
if(mapName.contains("-") && mapName.endsWith("楼")){
mapName = mapName.substring(0, mapName.indexOf("-"))
}
if(d['C']==mapName){
def q = new BasicDBObject()
q.put("city", d['A'])
q.put("region", d['E'])
q.put("name", d['C'])
q.put("cat", 'district')
q.put("status", 0)
def dbo = DbUtils.seaweed.findOne(q)
if(!dbo){
q.put("region", d['B'])
def lng2 = null
def lat2 = null
def lng = null
def lat = null
if(d['D']=="高德"){
lng = (d['I'].toDouble()*10000000000).longValue()
lat = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.getGaodeToBaidu(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng2 = (gmap.get("lng")*10000000000).longValue()
lat2 = (gmap.get("lat")*10000000000).longValue()
}
}else{
lng2 = (d['I'].toDouble()*10000000000).longValue()
lat2 = (d['J'].toDouble()*10000000000).longValue()
def gmap = GeocodeUtils.geoconvBaiduToGoogle(d['I'].toDouble(), d['J'].toDouble())
if(gmap){
lng = (gmap.get("lng")*10000000000).longValue()
lat = (gmap.get("lat")*10000000000).longValue()
}
}
def udata = new BasicDBObject()
udata.put("region", d['E'])
udata.put("lng", lng)
udata.put("lat", lat)
udata.put("lng2", lng2)
udata.put("lat2", lat2)
udata.put("m_date", new Date())
DbUtils.seaweed.update(q, new BasicDBObject('$set', udata))
d['K'] = "修正区域"
}
}
}
println index
},0)
}
@@ -236,7 +314,7 @@ class LocationTest {
}
}
def baiduRest = FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
def baiduRest = null;//FangjiaClientUtils.searchMapData(dist.city, dist.name,"住宅", "百度")
if(baiduRest){
bdlocationMap = getMapLocation(dist, baiduRest);
if(bdlocationMap){
@@ -280,7 +358,7 @@ class LocationTest {
region = r.district
address = r.address
if(r.district.contains(dist.region)){
if(r.district && r.district.contains(dist.region)){
score = 10
}else {
score = 5
@@ -303,7 +381,7 @@ class LocationTest {
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
if(r.district && r.district.contains(dist.region)){
score = 9
}else {
score = 5
@@ -320,7 +398,7 @@ class LocationTest {
region = r.district
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
if(r.district && r.district.contains(dist.region)){
score = 8
}else {
score = 5
@@ -338,7 +416,7 @@ class LocationTest {
region = r.district
mapName = r.name
address = r.address
if(r.district.contains(dist.region)){
if(r.district && r.district.contains(dist.region)){
score = 7
}else {
score = 5
@@ -366,15 +444,85 @@ class LocationTest {
return flag;
}
def findRegionRange(){
def findRegionRange(city){
def q = new BasicDBObject()
q.put("city","福州市")
q.put("city",city+"市")
def dbcursor = POLYLINECOLL.find(q)
def regionMap = [:]
for(def d : dbcursor){
def region = RegionData.formatRegion("福州",d.region)
def region = RegionData.formatRegion(city,d.region)
regionMap.put(region, d.polyline)
}
return regionMap
}
def run5(){
def results = getAllErrorDistricts()
def rets = Collections.synchronizedList(new ArrayList());
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(results.size())
for(def i=0;i<results.size();i++){
final def map = results[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = mapLocation(map)
if(ret){
rets << ret
}
println totalCount.get()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("d:\\数据\\福州\\经纬度错误.xlsx", rets, "经纬度错误")
}
def mapLocation(map){
def dist = CommonDatautils.findDistrictById(map[1])
if(dist){
def loactionMap = matchMap(dist)
def ret = []
ret << dist.city
ret << dist.region
ret << dist.name
if (loactionMap) {
ret << loactionMap.source
ret << loactionMap.region
ret << loactionMap.mapName
ret << loactionMap.address
def distance = GeoHelper.GetDistance(dist.lat / 10000000000.0, dist.lng / 10000000000.0, loactionMap.lat, loactionMap.lng)
ret << distance
ret << loactionMap.lng
ret << loactionMap.lat
if (dist.region == loactionMap.region) {
println dist.region + " " + dist.name + " 不在区域范围内,与地图相同"
} else {
println dist.region + " " + dist.name + " 不在区域范围内 地图:" + loactionMap.region + " " + distance
}
}
return ret
}
return null
}
def getAllErrorDistricts(){
def sql = "select cityid,commid from bigdata_community_distcheck where distname<>checkdist or checkdist is null";
def rets = []
sqlInstance.eachRow(sql,{r->
rets << [r.cityid, r.commid]
});
return rets
}
}
@@ -0,0 +1,27 @@
package com.fangjia.script.gj.location
class Point {
Double lat;
Double lng;
public Point(Double lng, Double lat){
this.lat = lat
this.lng = lng
}
Double getLat() {
return lat
}
void setLat(Double lat) {
this.lat = lat
}
Double getLng() {
return lng
}
void setLng(Double lng) {
this.lng = lng
}
}
@@ -0,0 +1,143 @@
package com.fangjia.script.gj.location
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.awt.geom.Point2D
class PointUtil {
public static final DBCollection BUSINESS_RANGE_COLL = MongoUtils.getCollectionByName("fangjia_crawl", "price_border", "mongo.db.fangjia.com")
/**
* 判断当前位置是否在多边形区域内
*
* @param checkPoint 当前经纬度
* @param areaAround 围栏范围
* @return
*/
public static boolean checkIsInPolygon(Point checkPoint, List<Point> areaAround) {
Point2D.Double point = new Point2D.Double(checkPoint.getLat(), checkPoint.getLng());
List<Point2D.Double> pointList = new ArrayList<>();
for (Point everyPoint : areaAround) {
Point2D.Double polygonPoint = new Point2D.Double(everyPoint.getLat(), everyPoint.getLng());
pointList.add(polygonPoint);
}
return checkIsPtInPoly(point, pointList);
}
/**
* 判断点是否在多边形内,如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
*
* @param point 检测点
* @param pts 多边形的顶点
* @return 点在多边形内返回true, 否则返回false
*/
public static boolean checkIsPtInPoly(Point2D.Double point, List<Point2D.Double> pts) {
int N = pts.size();
//如果点位于多边形的顶点或边上,也算做点在多边形内,直接返回true
boolean boundOrVertex = true;
//cross points count of x
int intersectCount = 0;
//浮点类型计算时候与0比较时候的容差
double precision = 2e-10;
//neighbour bound vertices
Point2D.Double p1, p2;
//当前点
Point2D.Double p = point;
//left vertex
p1 = pts.get(0);
//check all rays
for (int i = 1; i <= N; ++i) {
if (p.equals(p1)) {
//p is an vertex
return boundOrVertex;
}
//right vertex
p2 = pts.get(i % N);
//ray is outside of our interests
if (p.x < Math.min(p1.x, p2.x) || p.x > Math.max(p1.x, p2.x)) {
p1 = p2;
//next ray left point
continue;
}
//ray is crossing over by the algorithm (common part of)
if (p.x > Math.min(p1.x, p2.x) && p.x < Math.max(p1.x, p2.x)) {
//x is before of ray
if (p.y <= Math.max(p1.y, p2.y)) {
//overlies on a horizontal ray
if (p1.x == p2.x && p.y >= Math.min(p1.y, p2.y)) {
return boundOrVertex;
}
//ray is vertical
if (p1.y == p2.y) {
//overlies on a vertical ray
if (p1.y == p.y) {
return boundOrVertex;
//before ray
} else {
++intersectCount;
}
} else {
//cross point on the left side
//cross point of y
double xinters = (p.x - p1.x) * (p2.y - p1.y) / (p2.x - p1.x) + p1.y;
//overlies on a ray
if (Math.abs(p.y - xinters) < precision) {
return boundOrVertex;
}
//before ray
if (p.y < xinters) {
++intersectCount;
}
}
}
} else {
//special case when ray is crossing through the vertex
//p crossing over p2
if (p.x == p2.x && p.y <= p2.y) {
//next vertex
Point2D.Double p3 = pts.get((i + 1) % N);
//p.x lies between p1.x & p3.x
if (p.x >= Math.min(p1.x, p3.x) && p.x <= Math.max(p1.x, p3.x)) {
++intersectCount;
} else {
intersectCount += 2;
}
}
}
//next ray left point
p1 = p2;
}
//偶数在多边形外
if (intersectCount % 2 == 0) {
return false;
} else {
//奇数在多边形内
return true;
}
}
public static void main(def a){
new PointUtil().run()
}
def run(){
def q = new BasicDBObject()
q.put("name", "淞南")
def obj = BUSINESS_RANGE_COLL.findOne(q)
if(obj){
def pointStr = obj.border
if(pointStr){
def pointArrs = pointStr.split(";")
def list = []
for(def s : pointArrs){
def arr = s.split(",")
list << new Point(arr[0].toDouble(), arr[1].toDouble())
}
println checkIsInPolygon(new Point(121.491478,31.327947), list)
}
}
}
}
@@ -1,5 +1,6 @@
package com.fangjia.script.gj.price
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.util.ExcelUtils
@@ -11,32 +12,35 @@ class EvaluationBatch {
task.run()
}
def run(){
def filePath = "D:\\数据\\农行\\东莞农行测试数据(御景花园18条估值)(人工询价).xlsx"
def filePath = "D:\\数据\\银联\\房价网测试数据.xlsx"
def city = "¶«Ý¸"
ExcelUtils.modify(filePath, { data ->
if(!data['M']){
// def address = data['C'].replace("\"","").trim()
// def rMap = SeaweedDataMatchProcess.doMatch(city, data['K'],data['L'], address );
def rMap = SeaweedDataMatchProcess.doMatch(city, data['D'],data['E'], null);
if(rMap){
def json = evaluation(rMap, data['H'], 7, data['I'],null)
def region = RegionData.formatRegion(data['A'], data['B'])
if(region){
def rMap = SeaweedDataMatchProcess.doMatch(data['A'], region, data['C'], null);
if(rMap){
def json = evaluation(rMap, data['D'], null, null,null)
// data['J'] = rMap.city
// data['K'] = rMap.region
// data['L'] = rMap.name
if(json){
def avgPrice = json.avg_price
def totalPrice = json.total_price
data['N'] = totalPrice
data['M'] = avgPrice
data['O'] = json.credit
data['P'] = rMap.region
data['Q'] = rMap.name
data['E'] = rMap.city
data['F'] = rMap.region
data['G'] = rMap.name
if(json){
def avgPrice = json.avg_price
def totalPrice = json.total_price
data['H'] = totalPrice
data['I'] = avgPrice
data['J'] = json.credit
// data['P'] = rMap.region
// data['Q'] = rMap.name
}
}
}
}
}, 0);
@@ -0,0 +1,44 @@
package com.fangjia.script.gj.price
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class NoPriceCalcutePrice {
DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new NoPriceCalcutePrice()
task.run()
}
def run(){
def citys = CcbCommonUtils.citys;
for(def city : citys){
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name"])
def map = getDistrictByCity(city, "2021-08-01")
for(def d : districts){
if(!map.containsKey(d.region+"@"+d.name)){
FangjiaClientUtils.sendDistrictPriceCalculate(d.city, d.region, d.name, 12);
println d.city+" "+d.region+" "+d.name
}
}
}
}
def getDistrictByCity(city, weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd"))
def dbcursor = DIST_SELL_STAT.find(q)
def map = [:]
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
return map
}
}
@@ -90,7 +90,7 @@ class AddressStandard {
}
}
}
},0)
},0, "Sheet1")
}
@@ -60,12 +60,12 @@ class CrawlDistrictIdSelect {
def findCrawlData(){
def q = new BasicDBObject()
// q.put("city","¸£ÖÝ")
q.put("source", "°²¾Ó¿Í")
q.put("city","福州")
q.put("source", "城市房产")
q.put("s_date",202012)
def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def retMap = [:]
def prefix = "anjuke.com/"
def prefix = ".cityhouse.cn/ha/"
def keyList =new HashSet()
for(def db : dbcursor){
def url = null
@@ -77,11 +77,11 @@ class CrawlDistrictIdSelect {
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = db.district_id//url.substring(url.indexOf(prefix)+prefix.length(),url.size()-1)
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html"))
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(".anjuke.com"))
def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix))
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
@@ -92,7 +92,7 @@ class CrawlDistrictIdSelect {
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "anjuke.com"
map.root = "cityhouse.com"
map.city_short= shortCity
map.city = db.city
map.region = db.region
@@ -1,7 +1,35 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import java.text.SimpleDateFormat
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date date = new Date(1617211269868);
print dateFormat.format(date);
//SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
//Date date = new Date(1617211269868);
//print dateFormat.format(date);
DBCollection CITY_INFO_CONN = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_data_ana').getCollection('city_info');
def path = "D:\\数据\\广发\\广东省小区及均价覆盖率统计.xlsx"
ExcelUtils.modify(path,{d->
def districts = CommonDatautils.findAllDistrictByCity(d['B'], "district", ["name"])
d['G'] = districts.size()
def q = new BasicDBObject()
q.put("city", d['B'])
q.put("weekend", DateUtils.parseFromTightYYYYMMDD("2021-08-08"))
def obj = CITY_INFO_CONN.findOne(q);
if(obj){
d['H'] = obj.price_coverage*100
}
})
@@ -35,17 +35,21 @@ class InsidePolygon {
public static List<Point> createPoint2(String points){
String[] array = points.split(";");
List<Point> list = new ArrayList<Point>();
for(int i =0 ;i<array.length-1;i++){
def pointArrs = array[i].split("[,|\\|]")
Point point = new Point();
double x = Double.parseDouble(pointArrs[0]);
double y = Double.parseDouble(pointArrs[1]);
point.setLon(x);
point.setLat(y);
list.add(point);
if(points){
String[] array = points.split(";");
for(int i =0 ;i<array.length-1;i++){
def pointArrs = array[i].split("[,|\\|]")
Point point = new Point();
double x = Double.parseDouble(pointArrs[0]);
double y = Double.parseDouble(pointArrs[1]);
point.setLon(x);
point.setLat(y);
list.add(point);
}
}
return list;
}
@@ -1,6 +1,7 @@
package com.fangjia.script.thirdPriceCrawl
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import org.apache.solr.common.util.Hash
@@ -13,8 +14,9 @@ class SynchDistrictPrice {
def static final SHORT_DATE = Utils.getLastMonth()
def thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def districtDetailColl = CrawalDBUtil.getDistrictDetailColl();
def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
"anjuke.com":"安居客"]
// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房",
// "anjuke.com":"安居客","cityhouse.cn":"城市房产"]
def sourceMap = ["zhuge.com":"诸葛找房"]
static main(args) {
def task = new SynchDistrictPrice()
task.run()
@@ -39,6 +41,7 @@ class SynchDistrictPrice {
def q = new BasicDBObject()
q.put("city", city)
q.put("root", root)
q.put('m_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-07-01")));
def keys = new BasicDBObject()
keys.put("city", 1)
keys.put("region", 1)
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202105);
def count2 = findTotalCount(source, 202106);
def count1 = findTotalCount(source, 202106);
def count2 = findTotalCount(source, 202107);
println source +" "+count1 +" "+count2
}
}
@@ -531,8 +531,10 @@ public class DataUtil {
if(!(address ==~ /.*(½»»ã|½»¿Ú|½»²æ¿Ú)$/)){
address += "½»»ã"
}
}else{
address = null
}
address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
// address = address.replaceAll("(½»»ã|½»¿Ú|½»²æ¿Ú|½»»ã´¦).*","½»»ã")
}
return address
}
@@ -921,13 +921,13 @@ class DateUtils {
public static String getRandomDate(def startDate, def endDate){
Date randomDate = randomDate(startDate, endDate);
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd 00:00:00");
String result = format.format(randomDate);
return result;
}
static main(args){
println getRandomDate("2021-08-01","2021-08-31")
// nowFirstDay();
// nowLastDay();
/*def weekend = DateUtil.formatByPattern(getFirstDayOfWeek(new Date()), "yyyy-MM-dd");
@@ -955,8 +955,9 @@ class DateUtils {
// Date randomDate = randomDate("2010-09-20", "2011-09-22");
// SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");
// String result = format.format(randomDate);
System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
// System.out.println(getRandomDate("2010-09-20", "2011-09-22"));
println DateUtils.getDayOfWeek(new Date())
}
@@ -60,7 +60,8 @@ public class ExcelUtils{
FileInputStream inp = null;
try {
inp = new FileInputStream(file);
Workbook workbook = WorkbookFactory.create(inp)
BufferedInputStream bufferedInputStream = new BufferedInputStream(inp, 1024 * 200)
Workbook workbook = WorkbookFactory.create(bufferedInputStream)
workbook.setForceFormulaRecalculation(true)
Sheet sheet = workbook.getSheetAt(sheetIndex);
//ÐкÅ
@@ -82,6 +83,9 @@ public class ExcelUtils{
}
}
ret.add(rowData);
if (ret.size() % 10000 == 0) {
println(ret.size())
}
}
} catch (IOException e) {
e.printStackTrace();
@@ -625,7 +625,7 @@ public class OpenApiClient {
params.put("token", TOKEN)
try {
JSONObject result = HttpRequestUtil.getJsonContent("http://open.fangjia.com/property/evaluate", params)
if (result.getInt("code") == 200) {
if (result && result.getInt("code") == 200) {
if (result.containsKey("result")) {
// if (result.get("result").get("credit") == 0) {
// return null
@@ -8,10 +8,10 @@ import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def city = "À³Îß"
def destCity = "¼ÃÄÏ"
def region = ""
def destRegion = "À³Îß"
def city = "쭤捺"
def destCity = "쭤捺"
def region = "섈적"
def destRegion = "촙쏠"
@@ -26,6 +26,7 @@ class GaoDeApiUtils {
Map<String, String> params = new HashMap<String, String>()
params.put("key", GAO_DE_AK)
params.put("location", location)
params.put("extensions","all")
String content
try {
content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8")