Merge branch 'feature/monitor' into script_develop

# Conflicts:
#	projects/fangjia-script/pom.xml
This commit is contained in:
沈健
2021-05-21 10:14:52 +08:00
92 changed files with 7563 additions and 4519 deletions
+1
View File
@@ -19,6 +19,7 @@
<modules>
<module>../../projects/fangjia-core</module>
<module>../../projects/fangjia-dal</module>
<module>../../projects/chafangtong-dal</module>
<module>../../projects/fangjia-service</module>
<module>../../projects/geluke-dal</module>
<module>../../projects/geluke-service</module>
+95
View File
@@ -0,0 +1,95 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: geluke
namespace: fangjia
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 1
maxUnavailable: 0
revisionHistoryLimit: 5
selector:
matchLabels:
project: geluke
app: fangjia
template:
metadata:
labels:
project: geluke
app: fangjia
spec:
imagePullSecrets:
- name: nexus-registry-secret
nodeSelector:
app: fangjia
containers:
- name: geluke
image: 192.168.0.19:9012/geluke:latest
imagePullPolicy: Always
resources:
limits:
memory: "3Gi"
requests:
memory: "2Gi"
env:
- name: JAVA_OPTS
value: "-Xmx3072m -XX:MaxPermSize=256m
-Dlog4j.configuration=file:/home/goojia/webapps/ROOT/WEB-INF/classes/log4j_linux.xml
-Dfangjia.overridePropertiesLocation=/home/goojia/goojiaconfig/geluke.properties"
readinessProbe:
tcpSocket:
port: 8080
initialDelaySeconds: 10
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 8080
initialDelaySeconds: 10
periodSeconds: 5
ports:
- containerPort: 8080
name: web
protocol: TCP
volumeMounts:
- name: host-time
mountPath: /etc/localtime
- name: goojiaconfig
mountPath: /home/goojia/goojiaconfig
volumes:
- name: host-time
hostPath:
path: /etc/localtime
- name: goojiaconfig
cephfs:
monitors:
- 192.168.0.208:6789
- 192.168.0.209:6789
- 192.168.0.210:6789
user: admin
path: /k8s/goojiaconfig
secretRef:
name: ceph-secret
readOnly: false
---
apiVersion: v1
kind: Service
metadata:
name: geluke
namespace: fangjia
labels:
project: geluke
app: fangjia
spec:
type: NodePort
selector:
project: geluke
app: fangjia
ports:
- port: 8080
targetPort: 8080
nodePort: 30010
-63
View File
@@ -1,63 +0,0 @@
/**
* Created by Administrator on 2017/5/9.
*/
import com.sun.org.apache.xerces.internal.impl.io.UTF8Reader;
import com.sun.xml.internal.txw2.Document;
import sun.nio.cs.ext.GB18030;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.URLConnection;
public class Test {
String fixture_Date = null;
String area = null;
String unit_Price = null;
String transaction_Price = null;
static String sendGet(String url) {
String result = "";
// 定义一个缓冲字符输入流
BufferedReader in = null;
try {
// 将string转成url对象
URL realUrl = new URL(url);
// 初始化一个链接到那个url的连接
URLConnection connection = realUrl.openConnection();
// 开始实际的连接
connection.connect();
// 初始化 BufferedReader输入流来读取URL的响应
in = new BufferedReader(new InputStreamReader(connection.getInputStream(),"gbk"));
// 用来临时存储抓取到的每一行的数据
String line;
while ((line = in.readLine()) != null)
{ // 遍历抓取到的每一行并将其存储到result里面
result += line;
}
Document document = Jsoup.parse(result, url);
parseListPage(document);
} catch (Exception e) {
System.out.println("发送GET请求出现异常!" + e);
e.printStackTrace(); }
// 使用finally来关闭输入流
finally {
try {
if (in != null) {
in.close(); }
} catch (Exception e2) {
e2.printStackTrace();
}
}
return result;
}
public static void main(String[] args) {
// 定义即将访问的链接
String url = "http://dl.goufang.com/xq/1062/6";
// 访问链接并获取页面内容
String result = sendGet(url);
System.out.println(result);
}
}
@@ -122,10 +122,8 @@ public class District implements SeaweedEntity {
private String houseStructure; //房屋结构
private String source;
private String propertyLife;
//
// //乡镇街道
// private String township;
//
//乡镇街道
private String township;
private String pinyin;
@@ -1873,14 +1871,14 @@ public class District implements SeaweedEntity {
this.mappingAlias = mappingAlias;
}
// @Column(name = "township")
// public String getTownship() {
// return township;
// }
//
// public void setTownship(String township) {
// this.township = township;
// }
@Column(name = "township")
public String getTownship() {
return township;
}
public void setTownship(String township) {
this.township = township;
}
+5
View File
@@ -159,6 +159,11 @@
<artifactId>jfreechart</artifactId>
<version>1.5.3</version>
</dependency>
<dependency>
<groupId>org.apache.xmlgraphics</groupId>
<artifactId>batik-codec</artifactId>
<version>1.11</version>
</dependency>
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-core</artifactId>
@@ -1705,6 +1705,25 @@ public class AddressController {
return mav;
}
public Object similarityAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空"));
return mav;
}
model.setAddress(model.getAddress_1());
String result1 = addressMatchProxyService.standard(convertModelToMap(model));
model.setAddress(model.getAddress_2());
String result2 = addressMatchProxyService.standard(convertModelToMap(model));
if (result1 != null && result2 != null) {
Map<String, Object> resultMap = addressService.addressSimilarity(result1, result2);
mav.setModel(BasicModel.buildOKModel().setResult(resultMap));
} else {
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
}
return mav;
}
public Object parseCityAction(AddressModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getProvince())) {
@@ -1750,6 +1769,7 @@ public class AddressController {
// matchParams.setUseMap(true);
matchParams.setUseAliyunApi(true);
// matchParams.setUseCrf(true);
matchParams.setContainsLocation(model.isContainsLocation());
Building building = addressParserService.standard(matchParams);
Address address = building.getSearchAddress();
@@ -1790,6 +1810,7 @@ public class AddressController {
}
}
dto.append("buildingNumber", address.getBuildingNumber());
dto.append("floor", address.getFloor());
dto.append("unitNumber", address.getUnitNumber());
dto.append("roomNumber", address.getRoomNumber());
dto.append("formattedAddress", address.getFormattedAddress());
@@ -131,7 +131,7 @@ import com.google.common.collect.Sets;
import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
//@Authority(role = Role.USER)
@Authority(role = Role.USER)
public class PropertyController {
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
@@ -4966,6 +4966,7 @@ public class PropertyController {
* @throws NumberFormatException
* @throws IllegalAccessException
*/
@Authority(role = Role.NONE)
public Object rentReportAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
@@ -5164,7 +5165,6 @@ public class PropertyController {
}
public Object evaluateRentAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
RestfulModelAndView mav = new RestfulModelAndView();
mav.setViewType("rest");
@@ -5255,13 +5255,6 @@ public class PropertyController {
}
public void setHouseLoudongService(HouseLoudongService houseLoudongService) {
this.houseLoudongService = houseLoudongService;
}
@@ -9,6 +9,7 @@ import com.fangjia.dal.mongo.model.Evaluation;
import com.fangjia.dal.mongo.model.RentDealPrice;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.open.APIConstant;
import com.fangjia.open.compatible.model.JsonModel;
import com.fangjia.open.dto.DistrictHouseDtao;
import com.fangjia.open.interceptors.AuthenticationBean;
@@ -26,11 +27,17 @@ import com.fangjia.service.mongo.fangjia.FileService;
import com.fangjia.service.mongo.fangjia.SeawaterService;
import com.fangjia.service.util.DistrictScoreUtil;
import com.lowagie.text.pdf.BaseFont;
import org.apache.batik.transcoder.Transcoder;
import org.apache.batik.transcoder.TranscoderException;
import org.apache.batik.transcoder.TranscoderInput;
import org.apache.batik.transcoder.TranscoderOutput;
import org.apache.batik.transcoder.image.PNGTranscoder;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.TermQuery;
import org.bson.types.ObjectId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sothis.core.util.CollectionUtils;
@@ -41,9 +48,8 @@ import org.xhtmlrenderer.pdf.ITextRenderer;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.io.*;
import java.lang.reflect.InvocationTargetException;
import java.text.ParseException;
import java.util.*;
@@ -81,10 +87,29 @@ public class CcbRentController {
return result;
}
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
String result = null;
HttpRequest httpRequest = new HttpRequest();
try {
Map<String, String> headers = new HashMap<String, String>();
// &__refresh_cache=1 ˢлº´æ
String url = "http://open.fangjia.com/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN;
Map<String, String> params = org.apache.commons.beanutils.BeanUtils.describe(model);
result = httpRequest.post(url, params, headers,"UTF-8");
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (HttpRequestException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
return result;
}
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
@Param(name = "name") String name, @Param(name = "address") String address,
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment) {
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) {
com.fangjia.open.model.PropertyEvaluationModel model = new com.fangjia.open.model.PropertyEvaluationModel();
model.setCity(city);
model.setDistrict(districtStr);
@@ -261,7 +286,8 @@ public class CcbRentController {
districtPriceMap.put("trends", trendsMap);
// 生成走势图图片,一遍转pdf
String tradesImgUrl = buildTrendImage(trendsMap, room);
//String tradesImgUrl = buildTrendImage(trendsMap, room);
String tradesImgUrl = chartImgUrl;
//成交的租房房源
List<RentDealPrice> rentTrades = rentDealPriceService.getRentDealPrices(district.getCity(), district.getRegion(), district.getName(), 5);
@@ -426,23 +452,54 @@ public class CcbRentController {
return evaluation;
}
public Object saveHighChartAsImageAction(HttpServletRequest request, HttpServletResponse response) {
String svg = request.getParameter("svg");
ByteArrayOutputStream out = null;
String url = null;
try {
svg = svg.replaceAll(":rect", "rect");
Transcoder t = new PNGTranscoder();
out = new ByteArrayOutputStream();
TranscoderInput input = new TranscoderInput(new StringReader(svg));
TranscoderOutput output = new TranscoderOutput(out);
t.transcode(input, output);
byte[] data = out.toByteArray();
String id = new ObjectId().toString();
fileService.save(id,"o", data, "image/png", "o");
url = "http://img0.fangjia.com/stc/fs/" + DataUtil.encodePicUrl(id, "o", "house");
} catch (TranscoderException e) {
e.printStackTrace();
} finally {
try {
out.close();
} catch (IOException e) {
e.printStackTrace();
}
}
JsonModel model = new JsonModel();
AutoCompleteBo bo = new AutoCompleteBo();
bo.setAddress(url);
model.setJsonModel(bo);
return model;
}
public void downloadAction(HttpServletResponse response, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String district,
@Param(name = "name") String name, @Param(name = "address") String address,
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment) throws IOException {
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) throws IOException {
response.setHeader("Content-disposition", "attachment; filename=report.pdf");
response.setContentType("application/pdf;charset=UTF-8");
response.setCharacterEncoding("UTF-8");
OutputStream os = null;
try {
os = response.getOutputStream();
ITextRenderer renderer = new ITextRenderer(35, 20);
ITextRenderer renderer = new ITextRenderer(35, 24);
ITextFontResolver fontResolver = renderer.getFontResolver();
String fontUrl = ActionContext.getContext().getServletContext().getRealPath("/") + "fonts" + File.separator + "simsun.ttc";
fontResolver.addFont(fontUrl, BaseFont.IDENTITY_H, BaseFont.NOT_EMBEDDED);
final String path = "http://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath();
String url = path + "/ccbRent/downloadPage?city=" + DataUtil.urlEncode(city) + "&district="+DataUtil.urlEncode(district)+"&name="+DataUtil.urlEncode(name);
url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment;
url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment+"&chartImgUrl="+chartImgUrl;
renderer.setDocument(url);
renderer.layout();
renderer.createPDF(os, true);
@@ -19,7 +19,24 @@ public class AddesssMatchModel extends FangjiaApiModel {
private String province;
private String debugName;
private boolean useMap;
private String address_1;
private String address_2;
public String getAddress_1() {
return address_1;
}
public void setAddress_1(String address_1) {
this.address_1 = address_1;
}
public String getAddress_2() {
return address_2;
}
public void setAddress_2(String address_2) {
this.address_2 = address_2;
}
public boolean isContainsLocation() {
return containsLocation;
}
@@ -2,23 +2,24 @@ package com.fangjia.open.util;
import com.fangjia.core.util.DataUtil;
import com.fangjia.service.mongo.fangjia.FileService;
import org.apache.batik.transcoder.Transcoder;
import org.apache.batik.transcoder.TranscoderException;
import org.apache.batik.transcoder.TranscoderInput;
import org.apache.batik.transcoder.TranscoderOutput;
import org.apache.batik.transcoder.image.JPEGTranscoder;
import org.apache.batik.transcoder.image.PNGTranscoder;
import org.bson.types.ObjectId;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartPanel;
import org.jfree.chart.ChartUtilities;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.axis.NumberAxis;
import org.jfree.chart.plot.CategoryPlot;
import org.jfree.chart.plot.PlotOrientation;
import org.jfree.chart.renderer.category.LineAndShapeRenderer;
import org.jfree.chart.ui.ApplicationFrame;
import org.jfree.data.category.DefaultCategoryDataset;
import org.jfree.ui.RefineryUtilities;
import java.awt.*;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.*;
import java.util.ArrayList;
import java.util.List;
@@ -99,6 +100,42 @@ public class LineChartsUtils {
return null;
}
public static void svg2ImgTest() {
String svg = "<svg height=\"200\" xmlns=\"http://www.w3.org/2000/svg\"><polygon points=\"10,10 75,150 150,60\" style=\"fill:#63BCF7;stroke:black;stroke-width:3\"/></svg>";
String type="image/png";
String filename="svg";
if (null != type && null != svg) {
svg = svg.replaceAll(":rect", "rect");
String ext = "";
Transcoder t = null;
if (type.equals("image/png")) {
ext = "png";
t = new PNGTranscoder();
}
else if (type.equals("image/jpeg")) {
ext = "jpg";
t = new JPEGTranscoder();
}
if (null != t) {
OutputStream out = null;
try {
out = new FileOutputStream("D:\\"+filename+"."+ext);
} catch (FileNotFoundException e) {
e.printStackTrace();
}
TranscoderInput input = new TranscoderInput(new StringReader(svg));
TranscoderOutput output = new TranscoderOutput(out);
try{
t.transcode(input, output);
System.out.println("out");
}
catch (TranscoderException e) {
e.printStackTrace();
}
}
}
}
/**
* 定义图表对象
* @param rowKey 曲线名,如一居室
@@ -18,22 +18,28 @@
body {
overflow: hidden;
overflow-y: scroll;
background: #f8f8f8;
background: white;
}
.main {
margin: 4px 8px 4px 8px;
background: #f8f8f8;
background: white;
}
.title {
position: relative;
color: #1E90FF;
font-size: 20px;
font-weight: bold;
}
.title1 {
font-size: 14px;
font-size: 18px;
font-weight: bold;
color: #1E90FF;
margin-top: 2px;
margin-bottom: 2px;
}
.p {
font-size: 12px;
font-size: 18px;
line-height: 30px;
}
.table {
width: 100%;
@@ -77,11 +83,11 @@
<div class="title1">2 评估时间</div>
<div class="p">
<span name="cal_date"></span><br/>
评估结果<br/>
楼盘名称:<span name="name"></span><br/>
楼盘地址:<span name="address"></span><br/>
租金估值(RMB):<span name="totalPrice"></span>元/月<br/>
大写(人民币):<span name="totalPriceUpper"></span><br/>
<b>评估结果</b><br/>
<b>楼盘名称</b>:<span name="name"></span><br/>
<b>楼盘地址</b>:<span name="address"></span><br/>
<b>租金估值(RMB)</b>:<span name="totalPrice"></span>元/月<br/>
<b>大写(人民币)</b>:<span name="totalPriceUpper"></span><br/>
</div>
</div>
<div>
@@ -91,20 +97,21 @@
<div>
<div class="title1">4 房产状况</div>
<div class="p">
详细地址:<span name="address"></span><br/>
楼盘名称:<span name="name"></span><br/>
室号:<span name="room"></span><br/>
朝向:<span name="direction"></span><br/>
建筑面积:<span name="size"></span>平米<br/>
所在楼层:<span name="floor"></span> 层 <span name="height"></span>总层数<br/>
电梯配套:<span name="hasLift"></span><br/>
装修情况:<span name="fitment"></span><br/>
建筑年代:<span name="buildYear"></span><br/>
<b>详细地址</b>:<span name="address"></span><br/>
<b>楼盘名称</b>:<span name="name"></span><br/>
<b>室号</b>:<span name="room"></span><br/>
<b>朝向</b>:<span name="direction"></span><br/>
<b>建筑面积</b>:<span name="size"></span>平米<br/>
<b>所在楼层</b>:<span name="floor"></span> 层 <span name="height"></span>总层数<br/>
<b>电梯配套</b>:<span name="hasLift"></span><br/>
<b>装修情况</b>:<span name="fitment"></span><br/>
<b>建筑年代</b>:<span name="buildYear"></span><br/>
</div>
</div>
<div>
<div class="title1">5 历史租金</div>
<div class="charts" id="districtRentTrendsChart"></div>
<input type="hidden" id="chartImgUrl"/>
</div>
<div>
<div class="title1">6 已租房源</div>
@@ -179,7 +186,8 @@
<script type='text/javascript' src='http://g.alicdn.com/sj/lib/zepto/zepto.js' charset='utf-8'></script>
<script type='text/javascript' src='http://g.alicdn.com/msui/sm/0.6.2/js/sm.js' charset='utf-8'></script>
<!-- 暂时注释 -->
<script type="text/javascript" src="http://cdn.hcharts.cn/highcharts/highcharts.js"></script>
<script type="text/javascript" src="http://cdn.highcharts.com.cn/highcharts/9.0.1/highcharts.js"></script>
<script type="text/javascript" src="https://code.highcharts.com.cn/highcharts/modules/exporting.js"></script>
<script type='text/javascript' src='../static/js/wx.js' charset='utf-8'></script>
<script type="text/javascript" src="../static/js/trend-charts.js"></script>
<script src="https://html2canvas.hertzen.com/dist/html2canvas.js"></script>
@@ -218,9 +226,11 @@
// jspdf.addFont('SourceHanSans-Normal.ttf', 'SourceHanSans-Normal', 'normal');
// jspdf.setFont('SourceHanSans-Light');
//console.log(jspdf.getFontList());
var chart;
$("#download").bind('click', function(){
var chartImgUrl = $('#chartImgUrl').val();
window.location.href = "download?city="+params.city+"&district="+params.district+"&name="+params.name+"&address="+params.address+"&size="+params.size+"&floor="+params.floor
+"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment;
+"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment+"&chartImgUrl=" + chartImgUrl;
// jspdf.html(document.body, {
// callback: function (doc) {
// doc.save("租金报告.pdf");
@@ -319,9 +329,9 @@
<td class="td01">' + d.city + '</td> \
<td class="td02">' + d.region + '</td> \
<td>' + d.districtName + '</td> \
<td>' + d.totalPrice + '</td> \
<td>' + d.area + '</td> \
<td>' + d.room + '</td> \
<td>' + (d.totalPrice != undefined ? d.totalPrice : '') + '</td> \
<td>' + (d.area != undefined ? d.area : '') + '</td> \
<td>' + (d.room != undefined ? d.room : '') + '</td> \
<td>' + hall + '</td> \
<td>' + fitment + '</td> \
<td>' + formatDate(d.tradeDate) + '</td> \
@@ -340,10 +350,10 @@
<td class="td01">' + d.city + '</td> \
<td class="td02">' + d.region + '</td> \
<td>' + d.districtName + '</td> \
<td>' + d.totalPrice + '</td> \
<td>' + d.area + '</td> \
<td>' + d.room + '</td> \
<td>' + d.hall + '</td> \
<td>' + (d.totalPrice != undefined ? d.totalPrice : '') + '</td> \
<td>' + (d.area != undefined ? d.area : '') + '</td> \
<td>' + (d.room != undefined ? d.room : '') + '</td> \
<td>' + (d.hall != undefined ? d.hall : '')+ '</td> \
<td>' + fitment + '</td> \
<td>' + date + '</td> \
</tr>');
@@ -359,9 +369,9 @@
$('#sameDistricts').append('<tr class=""> \
<td class="td01">' + d.city + '</td> \
<td class="td02">' + d.region + '</td> \
<td>' + d.districtName + '</td> \
<td>' + d.roomPrice + '</td> \
<td>' + d.room + '</td> \
<td>' + (d.districtName != undefined ? d.districtName:'' ) + '</td> \
<td>' + (d.roomPrice != undefined ? d.roomPrice:'') + '</td> \
<td>' + (d.room != undefined ? d.room:'') + '</td> \
<td>1</td> \
</tr>');
})
@@ -426,8 +436,9 @@
var config = {yaxis : yaxis};
createHighChartNow(renderId, data, config);
};
function createHighChartNow(renderId, data, config) {
var chart = new Highcharts.Chart({
chart = new Highcharts.Chart({
credits:{
enabled:true,
text: '',
@@ -446,6 +457,9 @@
renderTo: renderId,
type: 'spline'
},
exporting: {
enabled: false
},
title: {
text: data.title,
floating: false,
@@ -479,7 +493,26 @@
}
}
});
saveHighChartImg();
};
function saveHighChartImg() {
var svg = chart.getSVG();
$.ajax({
// 注意需要加..
url: "saveHighChartAsImage",
data: {
svg:svg
},
dataType: "json",
type: "post",
success: function (ret) {
if (ret.address != null) {
$('#chartImgUrl').val(ret.address)
}
}
})
}
$(function () {
init();
});
@@ -14,221 +14,191 @@
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
<link rel="stylesheet" href="../static/style/wx.css"/>
<link rel="stylesheet" href="../static/style/report.css"/>
<style type="text/css">
body {
overflow: hidden;
overflow-y: scroll;
background: #f8f8f8;
/** 特别注意,否则中文不生效 **/
font-family: SimSun;
}
.main {
margin: 4px 8px 4px 8px;
background: #f8f8f8;
}
.title {
position: relative;
color: #1E90FF;
}
.title1 {
font-size: 14px;
color: #1E90FF;
}
.p {
font-size: 12px;
}
.table {
width: 100%;
overflow-y: hidden;
overflow-x: scroll
}
.inline {
width: 550px;
border-spacing: 0;
text-align: left;
}
.table .inline th {
color: #999;
font-weight: normal;
line-height: 30px;
text-align: left;
}
.download {
float: right;
color: #3d4145;
border: solid 1px #333;
order-radius: 3px;
font-size: 12px;
line-height: 1rem;
width: 43px;
margin-top: 10px;
}
</style>
</head>
<body>
<div class="main">
<div class="title">
估值报告
<div class="content">
<div class="t1">
<em>1</em>估价结果
</div>
<div>
<div class="title1">1 评估申明</div>
<div class="p">建设银行遵循中立、客观、公正的原则,通过周密细致的测算,评估求得分析对象的房地产市场参考。</div>
<div class="st1"><em>1.1</em>评估申明:</div>
<div class="text1">建设银行遵循中立、客观、公正的原则,通过周密细致的测算,评估求得分析对象的房<br/>地产市场参考。</div>
<div class="st1"><em>1.2</em>评估时间:</div>
<div class="text1"><#if result.cal_date??>${result.cal_date}</#if></div>
<div class="st1"><em>1.3</em>评估结果:</div>
<div class="text2">
<b>楼盘名称</b>:<#if result.name??>${result.name}</#if><br/><br/>
<b>楼盘地址</b>:<#if result.address??>${result.address}</#if><br/><br/>
<b>租金估值(RMB)</b>:<#if result.totalPrice??>${result.totalPrice}</#if>元/月<br/><br/>
<b>大写(人民币)</b>:<#if result.totalPriceUpper??>${result.totalPriceUpper}</#if><br/><br/>
</div>
<div class="st1"><em>1.4</em>报告有效期:</div>
<div class="text1" style="margin-bottom: 30px">本报告在市场无较大波动的情况下,该评估价格在报告出具起三个月内有效。</div>
<div class="t1"><em>2</em>房产状况</div>
<div class="c1">
<table class="distTab" width="700" border="0" cellspacing="0" cellpadding="0">
<tbody>
<tr>
<td class="textTd" style="width: 15%;">楼盘名称</td>
<td class="textRight1" style="width: 35%;"><#if result.name??>${result.name}</#if></td>
<td class="textTd" style="width: 15%;">地址</td>
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.address??>${result.address}</#if></td>
</tr>
<tr class="tdbg">
<td class="textTd">室号</td>
<td class="textRight1"><#if result.room??>${result.room}</#if></td>
<td class="textTd">朝向</td>
<td class="textRight2" colspan="3"><#if result.direction??>${result.direction}</#if></td>
</tr>
<tr>
<td class="textTd" style="width: 15%;">建筑面积</td>
<td class="textRight1" style="width: 35%;"><#if result.size??>${result.size}</#if>平米</td>
<td class="textTd" style="width: 15%;">所在楼层</td>
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.floor??>${result.floor}</#if> 层 <#if result.height??>${result.height}</#if>总层数</td>
</tr>
<tr class="tdbg">
<td class="textTd">电梯配套</td>
<td class="textRight1"><#if result.hasLift??>${result.hasLift}</#if></td>
<td class="textTd">装修情况</td>
<td class="textRight2" colspan="3"><#if result.fitment??>${result.fitment}</#if></td>
</tr>
<tr class="tdbg">
<td class="textTd">建筑年代</td>
<td class="textRight1"><#if result.buildYear??>${result.buildYear}</#if></td>
</tr>
</tbody>
</table>
</div>
<div class="t1">
<em>3</em>历史租金
</div>
<div>
<div class="title1">2 评估时间</div>
<div class="p">
<span name="cal_date"><#if result.cal_date??>${result.cal_date}</#if></span><br/>
评估结果<br/>
楼盘名称:<span name="name"><#if result.name??>${result.name}</#if></span><br/>
楼盘地址:<span name="address"><#if result.address??>${result.address}</#if></span><br/>
租金估值(RMB):<span name="totalPrice"><#if result.totalPrice??>${result.totalPrice}</#if></span>元/月<br/>
大写(人民币):<span name="totalPriceUpper"><#if result.totalPriceUpper??>${result.totalPriceUpper}</#if></span><br/>
</div>
<#if result.tradesImgUrl??>
<#-- <img width="800px" height="300px" src="../static/images/test.png"></img>-->
<img width="600px" height="300px" src="${result.tradesImgUrl}"></img>
<#else>
<div class="p table">
<table class="inline">
<tbody><tr><td>暂无数据</td></tr></tbody>
</table>
</div>
</#if>
</div>
<div>
<div class="title1">3 报告有效期</div>
<div class="p">本报告在市场无较大波动的情况下,该评估价格在报告出具起三个月内有效。</div>
<div class="t1">
<em>4</em>已租房源
</div>
<div>
<div class="title1">4 房产状况</div>
<div class="p">
详细地址:<span name="address"><#if result.address??>${result.address}</#if></span><br/>
楼盘名称:<span name="name"><#if result.name??>${result.name}</#if></span><br/>
室号:<span name="room"><#if result.room??>${result.room}</#if></span><br/>
朝向:<span name="direction"><#if result.direction??>${result.direction}</#if></span><br/>
建筑面积:<span name="size"><#if result.size??>${result.size}</#if></span>平米<br/>
所在楼层:<span name="floor"><#if result.floor??>${result.floor}</#if></span> 层 <span name="height"><#if result.height??>${result.height}</#if></span>总层数<br/>
电梯配套:<span name="hasLift"><#if result.hasLift??>${result.hasLift}</#if></span><br/>
装修情况:<span name="fitment"><#if result.fitment??>${result.fitment}</#if></span><br/>
建筑年代:<span name="buildYear"><#if result.buildYear??>${result.buildYear}</#if></span><br/>
</div>
</div>
<div>
<div class="title1">5 历史租金</div>
<div>
<#if result.tradesImgUrl??>
<#-- <img width="800px" height="300px" src="../static/images/test.png"></img>-->
<img width="800px" height="300px" src="${result.tradesImgUrl}"></img>
<#else>
<div class="p table">
<table class="inline">
<tbody><tr><td>暂无数据</td></tr></tbody>
</table>
</div>
</#if>
</div>
</div>
<div>
<div class="title1">6 已租房源</div>
<div class="p table">
<table class="inline">
<thead>
<tr class="bg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>成交价</th>
<th>面积</th>
<th>室</th>
<th>厅</th>
<th>装修</th>
<th>成交日期</th>
<div style="margin-bottom: 30px;">
<table class="distTab" width="780" border="0" cellspacing="0" cellpadding="0">
<thead>
<tr class="tdbg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>成交价</th>
<th>面积</th>
<th>室</th>
<th>厅</th>
<th>装修</th>
<th>成交日期</th>
</tr>
</thead>
<tbody>
<#if (result.trades??&&result.trades?size>0)>
<#list result.trades as bo>
<tr class="tdbg">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.area??>${bo.area}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}</#if></td>
<td><#if bo.fitment??>${bo.fitment}</#if></td>
<td><#if bo.source??>${bo.source}</#if></td>
</tr>
</thead>
<tbody>
<#if (result.trades??&&result.trades?size>0)>
<#list result.trades as bo>
<tr class="">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.area??>${bo.area}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}</#if></td>
<td><#if bo.fitment??>${bo.fitment}</#if></td>
<td><#if bo.source??>${bo.source}</#if></td>
</tr>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
<div>
<div class="title1">7 样本参考</div>
<div class="p table">
<table class="inline">
<thead>
<tr class="bg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>租金</th>
<th>面积</th>
<th>室</th>
<th>厅</th>
<th>装修</th>
<th>成交日期</th>
</tr>
</thead>
<tbody id="referenceHouses">
<#if (result.houses??&&result.houses?size>0)>
<#list result.houses as bo>
<tr class="">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.area??>${bo.area}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}</#if></td>
<td><#if bo.fitment??>${bo.fitment}</#if></td>
<td><#if bo.root??>${bo.root}</#if></td>
</tr>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
<div class="t1">
<em>5</em>样本参考
</div>
<div>
<div class="title1">8 周边同类小区租金</div>
<div class="p table">
<table class="inline" style="width: 300px">
<thead>
<tr class="bg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>平米租金</th>
<th>室</th>
<th>厅</th>
</tr>
</thead>
<tbody id="sameDistricts">
<#if (result.districts??&&result.districts?size>0)>
<#list result.districts as bo>
<tr class="">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}</#if></td>
</tr>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
<div style="margin-bottom: 30px;">
<table class="distTab" width="780" border="0" cellspacing="0" cellpadding="0">
<thead>
<tr class="tdbg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>租 金</th>
<th>面积</th>
<th>室</th>
<th>厅</th>
<th>装修</th>
<th>成交日期</th>
</tr>
</thead>
<tbody id="referenceHouses">
<#if (result.houses??&&result.houses?size>0)>
<#list result.houses as bo>
<tr class="tdbg">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.area??>${bo.area}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}</#if></td>
<td><#if bo.fitment??>${bo.fitment}</#if></td>
<td><#if bo.root??>${bo.root}</#if></td>
</tr>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
<div class="t1">
<em>6</em>周边同类小区租金
</div>
<div style="margin-bottom: 30px;">
<table class="distTab" width="500" border="0" cellspacing="0" cellpadding="0">
<thead>
<tr class="tdbg">
<th class="th01">城市</th>
<th>区域</th>
<th>小区名</th>
<th>平米租金</th>
<th>室</th>
<th>厅</th>
</tr>
</thead>
<tbody id="sameDistricts">
<#if (result.districts??&&result.districts?size>0)>
<#list result.districts as bo>
<tr class="tdbg">
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
<td><#if bo.districtName??>${bo.districtName}</#if></td>
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
<td><#if bo.room??>${bo.room}</#if></td>
<td><#if bo.hall??>${bo.hall}<#else>1</#if></td>
</tr>
</#list>
<#else>
<tr><td>暂无数据</td></tr>
</#if>
</tbody>
</table>
</div>
</div>
</body>
@@ -260,7 +260,7 @@
$("#evaluateBtn").prop("disabled", true)
$.ajax({
// 注意需要加..
url: '../property/evaluateRent',
url: '../ccbRent/evaluateRent',
data:$("#evaluateForm").serialize(),
dataType:"json",
type:"post",
Binary file not shown.

After

Width:  |  Height:  |  Size: 3.1 KiB

@@ -0,0 +1,141 @@
body {
overflow: hidden;
overflow-y: scroll;
background: white;
/** 特别注意,否则中文不生效 **/
font-family: SimSun;
}
.content {
width: 840px;
height: 100%;
margin: 0 auto;
background: #fff;
padding-left: 20px;
}
.content .t1 {
background: url(../images/fjt/bg.png) 0 0 no-repeat;
width: 820px;
height: 42px;
color: #704f1c;
font-size: 20px;
font-weight: bold;
text-indent: 14px;
line-height: 46px;
margin-bottom: 25px;
}
.content .t1 em {
font-family: "Arial";
color: #FFF;
margin-right: 20px;
}
em {
font-style: normal;
}
.content .st1 {
background: url(../images/fjt/bg.png) 0 -60px no-repeat;
width: 172px;
height: 40px;
color: #704f1c;
font-size: 18px;
font-weight: bold;
text-indent: 7px;
line-height: 46px;
margin-bottom: 10px;
}
.content .st1 em {
font-family: "Arial";
color: #FFF;
margin-right: 14px;
}
.content .text1 {
font-size: 18px;
line-height: 26px;
margin: 0 20px 10px 50px;
word-wrap: break-word;
}
.text1 td {
text-align: center
}
.content .text2 {
font-size: 18px;
line-height: 40px;
padding: 10px 20px 10px 50px;
background: #eee;
width: 750px;
margin-bottom: 30px;
}
.content .text3 {
font-size: 16px;
line-height: 40px;
padding: 10px 20px 10px 50px;
background: #eee;
width: 750px;
margin-bottom: 30px;
}
.content .c1 {
height: 200px;
}
.content .distTab {
font-size: 18px
}
.content .distTab .textTd {
font-weight: bold;
}
.content .distTab tr td {
height: 30px;
}
.content .distTab tr th {
text-align: left;
}
.content .distTab .textRight1 {
text-align: right;
padding-right: 80px;
}
.content .distTab .textRight2 {
text-align: right;
padding-right: 5px;
}
.title {
position: relative;
color: #1E90FF;
}
.title1 {
font-size: 19px;
color: #1E90FF;
margin-top: 3px;
margin-bottom: 3px;
}
.p {
font-size: 16px;
}
table {
border-collapse: separate;
text-indent: initial;
border-spacing: 2px;
}
tbody {
display: table-row-group;
vertical-align: middle;
border-color: inherit;
}
.inline {
width: 550px;
border-spacing: 0;
text-align: left;
}
.table .inline th {
color: #999;
font-weight: normal;
line-height: 30px;
text-align: left;
}
.download {
float: right;
color: #3d4145;
border: solid 1px #333;
order-radius: 3px;
font-size: 12px;
line-height: 1rem;
width: 43px;
margin-top: 10px;
}
@@ -32,6 +32,7 @@ public enum Watcher {
BaYi('15300424133','bayi@fangjia.com'),
WeiLei('18101751850','weilei@fangjia.com'),
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
ShenJian('13764369683', 'shenjian@fangjia.com')
private final String mobile;
private final String email;
@@ -218,7 +218,7 @@ class CommonDatautils {
q.put("visible", 0);
q.put("cat", cat);
q.put("city", city);
// q.put("name", "工商大楼");
// q.put("name","¶«Ô¢Ð´å")
DBObject keys = new BasicDBObject();
if(fields){
@@ -1392,6 +1392,21 @@ class CommonDatautils {
}
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
DBObject queryCondition = new BasicDBObject()
queryCondition.put("city", city);
queryCondition.put("region", region);
queryCondition.put("district_name", name);
if(weekend){
queryCondition.put('weekend', new BasicDBObject('$gte', weekend));
}
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit)
def dbo = null;
while(cur.hasNext()) {
dbo = cur.next()
}
return dbo;
}
@@ -0,0 +1,148 @@
package com.fangjia.script.companyCrawl
import net.sf.json.JSONObject
import org.jsoup.Jsoup
import java.util.concurrent.atomic.AtomicInteger
class AiqichaCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
public final static int AbuyunMaxCount = 10;
static main(args) {
def task = new AiqichaCrawl();
task.process()
}
void process(Object args) {
def url = "https://aiqicha.baidu.com/s/l?q=%E6%96%B0%E5%8F%8B&t=&p=3&s=10&o=0&f=%7B%7D "
for(def key : companyKeys){
def queryBody = null
def s = crawl(url, queryBody);
if(s){
print s
}
println s
}
}
def crawl(url, queryBody){
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","aiqicha.baidu.com")
headers.put("Connection","keep-alive")
headers.put("Cache-Control","max-age=0")
headers.put("sec-ch-ua",'"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"')
headers.put("sec-ch-ua-mobile","?0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Sec-Fetch-Site","same-origin")
headers.put("Sec-Fetch-Mode","navigate")
headers.put("Sec-Fetch-User","?1")
headers.put("Sec-Fetch-Dest","document")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
String s = doPost(url, queryBody, headers)
if(s && s.contains("window.pageData = ")){
def prefix = "window.pageData ="
def s1 = s.substring(s.indexOf(prefix)+prefix.size())
s1 = s1.substring(0, s1.indexOf("}};")+2)
JSONObject jsonObject = JSONObject.fromObject(s1);
print jsonObject
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -0,0 +1,715 @@
package com.fangjia.script.companyCrawl
import com.eclipsesource.v8.V8
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.util.ABuYunUtil
import jdk.nashorn.api.scripting.ScriptObjectMirror
import org.apache.commons.io.IOUtils;
import org.apache.http.Header
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet
import org.jsoup.Jsoup
import org.jsoup.select.Elements
import org.apache.http.impl.client.HttpClients
import javax.script.ScriptEngine
import javax.script.ScriptEngineManager;
import java.util.concurrent.atomic.AtomicInteger
class GsxtCrawl {
def static final proxyPort = 9020;
def static final proxyServer = "http-dyn.abuyun.com";
def static final proxyUser = "HZU1XU28H3G4Z44D";
def static final proxyPass = "466C86ED77FE43D4";
public final static int AbuyunMaxCount = 10;
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
static main(args) {
def task = new GsxtCrawl();
//task.process()
task.run()
}
def run(){
getObject()
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
get.setHeader("Upgrade-Insecure-Requests", "1");
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
HttpResponse response = client.execute(get);
def __jsluid_h = getJsluid(response)
def s = getResponseBodyAsString(response)
println (s)
def __jsl_clearance = getJsContent(s);
//Cookie: __jsluid_h=26cb8385493c2e424767eca767720c87; __jsl_clearance=1618899529.627|-1|ao39lP6IVSloPMvq5%2FOSBqSiPZ0%3D
def newCookie = __jsluid_h+";"+__jsl_clearance
get.setHeader("Cookie", newCookie)
HttpResponse response2 = client.execute(get);
def s2 = getResponseBodyAsString(response2)
def __jsl_clearance2 = getJsContent2(s2);
println(__jsl_clearance2)
// fristCrawl()
}
//该函数测试Java与js对象
static void getObject() throws Exception{
String str = "var _0xbe8c = ['w4LCg8Onw7o=', 'ZAjDvCw=', 'YcKOw6Fm', 'wpHCqsO3CQ==', 'UMKCXnw=', 'w7Aww6XCnQ==', 'BcKNwrvCng==', 'J2zCnsKi', 'wq02WMOx', 'RQXDrUw=', 'eCnCnDw=', 'Q8KowpzDjA==', 'YznDucOn', 'YcOrXxk=', 'w4t9PsKM', 'DMO4QmA=', 'w5HCklPDmQ==', 'w5rCv8Omw6Q=', 'NcOwc8K5', 'GUxjw7Q=', 'w7zDicK8w4I=', 'VCDDpsKv', 'cwLChT4=', 'wpvCl1vDqw==', 'wrvCvMOKNQ==', 'RTnDrAc=', 'GMKFw4hk', 'SxnDvC0=', 'H2NHNQ==', 'w6Aqw7LCqg==', 'wq0mW8Ok', 'L1zCv8Kp', 'B8OqwoI2', 'Jl7DuX8=', 'JcOBwpIR', 'U8OnfSo=', '6K2K5rOG6aqL6KyM', 'f8Kiwo/Dhg==', 'aMK6w6vDog==', 'JcKgw4FC', 'w61vJEg=', 'LMOrwoY0', 'GcK4w6lt', 'w4XDrsO5wpY=', 'J8OhwoM1', 'w5Mzw7J/', 'wocbcBA=', 'SibDt8Oj', 'w7xgwoXDkQ==', 'AcKyDEQ=', 'Rg7Dkw==', 'w6UFK8Kw', 'TTzDjcKK', 'w6QhDcKy', 'ZBLDols=', 'ZjPDrk4=', 'w6XDqX3Ctw==', 'w6EvJcKT', 'JHtUDQ==', 'wq8mI8KE', 'JMOhwoJ9', 'MsK1AkE=', 'w7fCmhcm', 'VsK2w5fDtQ==', 'w7EUw4PCnw==', 'dz/DgsKu', 'UzfDtcK0', 'wpd9fWE=', 'worDvcKFw4M=', 'w4PDr0zDhg==', 'w7ocSMKg', 'wp1uwoZd', 'eAjDv8Oz', 'aATCgTw=', 'w7UGw7k=', 'wqZCwotq', 'HFIBw5s=', 'WsOdw7JB', 'w5PDpXHCpA==', 'GMO8wqdD', 'w4TCsQs=', 'w6/DmcODwpg=', 'w7MuwrjDoQ==', 'dDHDhsOj', 'G308w70=', 'CsO6wog9', 'FcONwrVw', 'M3vCmMK4', 'w6/Dm8OvwpI=', 'QsOLw5ZM', 'TgLDicKy', 'QSHCuQA=', 'UsOlw5RI', 'w7EqwqDDrw==', 'w6jDusK5w4k=', 'OsKaw5rDlg==', 'wosLQsOk', 'DcKxw7Vo', 'w5VuIEo=', 'w7sXw6TCqw==', 'wrLDr8KRw7s=', 'wr9+wpJR', 'w4LDscO0wrQ=', 'FFIPw6U=', 'J8Ksw5rDlA==', 'd8KZwqjDgw==', 'PcKvDF4=', 'MHh2GA==', 'dgDDs0Y=', 'wpxSS1Y=', 'w7l5JMKh', 'w7N8JMKC', 'XsKhw7hR', 'QxDDqm4=', 'PcKGw5ZJ', 'w7HCuAIQ', 'E8Kmw53Dmg==', 'CMOrwok3', 'U8K9wpvDnA==', 'IW06w5U=', 'w7DDjcOGwpQ=', 'X8KlwpHDpA==', 'w6TDn8O1wpo=', 'ZjnDv08=', 'w7/Cqjsp', 'wqLDk8KYw58=', 'w6fDvMKXw4g=', 'FsK8IUY=', 'w4knw4Rh', 'w5zCu8Ogw5M=', 'wqJ0ckc=', 'T8Kgw69G', 'GF0hw74=', 'FMOkwqkH', 'wokmbMOM', 'djHDtcKW', 'FXkow64=', 'B8OHwowH', 'VSjCpQU=', 'w5vDs8K0w4s=', 'w7vCkUV1', 'w7dMIcK8', 'WMOjZyI=', 'LcKBw7po', 'BMK7HkY=', 'B8K0w7F4', 'YsK+w67Drw==', 'HlQKw7I=', 'wpLCrMKrNg==', 'w7IjcsKy', 'w6zDusKhw4o=', 'WhPDm8Om', 'dMKlw7RO', 'dcOHw5R3', 'Nkpww7M=', 'wrnCsMKLdg==', 'dDTDqMKb', 'S8KXZXE=', 'wq/CssOpJA==', 'w7J/KFw=', 'QMKSWEo=', 'woNxbEc=', 'wqVIwow=', 'TD/Dqg4=', 'w7TDpmzCiw==', 'w7HDjWDCgA==', 'w7IEw6vCoA==', 'Q8K+w4zDiQ==', 'PElvw7k=', 'cDXDsRo=', 'FMOLX0c=', 'Z8KHwojDoA==', 'TTTCqzo=', 'woXCrsK6Uw==', 'w4lhHnY=', 'w6Q/dsKj', 'KsKIwpLCtQ==', 'QQTDn8Oy', 'D8Omw7zDoQ==', 'E8KzGk0=']; (function(_0x44c192, _0xbe8c61) {\n" +
"\t\tvar _0x1533fb = function(_0x5b499f) {\n" +
"\t\t\twhile (--_0x5b499f) {\n" +
"\t\t\t\t_0x44c192['push'](_0x44c192['shift']());\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\t_0x1533fb(++_0xbe8c61);\n" +
"\t} (_0xbe8c, 0x185));\n" +
"\tvar _0x1533 = function(_0x44c192, _0xbe8c61) {\n" +
"\t\t_0x44c192 = _0x44c192 - 0x0;\n" +
"\t\tvar _0x1533fb = _0xbe8c[_0x44c192];\n" +
"\t\tif (_0x1533['GlBvhA'] === undefined) { (function() {\n" +
"\t\t\t\tvar _0xa131c3 = function() {\n" +
"\t\t\t\t\tvar _0x9829e5;\n" +
"\t\t\t\t\ttry {\n" +
"\t\t\t\t\t\t_0x9829e5 = Function('return\\x20(function()\\x20' + '{}.constructor(\\x22return\\x20this\\x22)(\\x20)' + ');')();\n" +
"\t\t\t\t\t} catch(_0x1ba3fa) {\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t\treturn _0x9829e5;\n" +
"\t\t\t\t};\n" +
"\t\t\t\tvar _0x5e86d6 = _0xa131c3();\n" +
"\t\t\t\tvar _0x67fb0c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';\n" +
"\t\t\t\t_0x5e86d6['atob'] || (_0x5e86d6['atob'] = function(_0x1cb044) {\n" +
"\t\t\t\t\tvar _0xa69f93 = String(_0x1cb044)['replace'](/=+\$/, '');\n" +
"\t\t\t\t\tvar _0x26bfb9 = '';\n" +
"\t\t\t\t\tfor (var _0x2f1d79 = 0x0,\n" +
"\t\t\t\t\t_0x3713bb, _0x4e514b, _0x29733b = 0x0; _0x4e514b = _0xa69f93['charAt'](_0x29733b++);~_0x4e514b && (_0x3713bb = _0x2f1d79 % 0x4 ? _0x3713bb * 0x40 + _0x4e514b: _0x4e514b, _0x2f1d79++%0x4) ? _0x26bfb9 += String['fromCharCode'](0xff & _0x3713bb >> ( - 0x2 * _0x2f1d79 & 0x6)) : 0x0) {\n" +
"\t\t\t\t\t\t_0x4e514b = _0x67fb0c['indexOf'](_0x4e514b);\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t\treturn _0x26bfb9;\n" +
"\t\t\t\t});\n" +
"\t\t\t} ());\n" +
"\t\t\tvar _0x38a7ff = function(_0x5dd34b, _0x3a0f2e) {\n" +
"\t\t\t\tvar _0x3e7a99 = [],\n" +
"\t\t\t\t_0x3e2c47 = 0x0,\n" +
"\t\t\t\t_0x383663,\n" +
"\t\t\t\t_0x14720e = '',\n" +
"\t\t\t\t_0x5864f4 = '';\n" +
"\t\t\t\t_0x5dd34b = atob(_0x5dd34b);\n" +
"\t\t\t\tfor (var _0x28a6d2 = 0x0,\n" +
"\t\t\t\t_0x2ea622 = _0x5dd34b['length']; _0x28a6d2 < _0x2ea622; _0x28a6d2++) {\n" +
"\t\t\t\t\t_0x5864f4 += '%' + ('00' + _0x5dd34b['charCodeAt'](_0x28a6d2)['toString'](0x10))['slice']( - 0x2);\n" +
"\t\t\t\t}\n" +
"\t\t\t\t_0x5dd34b = decodeURIComponent(_0x5864f4);\n" +
"\t\t\t\tvar _0x3091fb;\n" +
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3091fb;\n" +
"\t\t\t\t}\n" +
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb] + _0x3a0f2e['charCodeAt'](_0x3091fb % _0x3a0f2e['length'])) % 0x100;\n" +
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
"\t\t\t\t}\n" +
"\t\t\t\t_0x3091fb = 0x0;\n" +
"\t\t\t\t_0x3e2c47 = 0x0;\n" +
"\t\t\t\tfor (var _0x8c68bc = 0x0; _0x8c68bc < _0x5dd34b['length']; _0x8c68bc++) {\n" +
"\t\t\t\t\t_0x3091fb = (_0x3091fb + 0x1) % 0x100;\n" +
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb]) % 0x100;\n" +
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
"\t\t\t\t\t_0x14720e += String['fromCharCode'](_0x5dd34b['charCodeAt'](_0x8c68bc) ^ _0x3e7a99[(_0x3e7a99[_0x3091fb] + _0x3e7a99[_0x3e2c47]) % 0x100]);\n" +
"\t\t\t\t}\n" +
"\t\t\t\treturn _0x14720e;\n" +
"\t\t\t};\n" +
"\t\t\t_0x1533['sMWqUz'] = _0x38a7ff;\n" +
"\t\t\t_0x1533['AbOzCI'] = {};\n" +
"\t\t\t_0x1533['GlBvhA'] = !![];\n" +
"\t\t}\n" +
"\t\tvar _0x5b499f = _0x1533['AbOzCI'][_0x44c192];\n" +
"\t\tif (_0x5b499f === undefined) {\n" +
"\t\t\tif (_0x1533['uRkTAI'] === undefined) {\n" +
"\t\t\t\t_0x1533['uRkTAI'] = !![];\n" +
"\t\t\t}\n" +
"\t\t\t_0x1533fb = _0x1533['sMWqUz'](_0x1533fb, _0xbe8c61);\n" +
"\t\t\t_0x1533['AbOzCI'][_0x44c192] = _0x1533fb;\n" +
"\t\t} else {\n" +
"\t\t\t_0x1533fb = _0x5b499f;\n" +
"\t\t}\n" +
"\t\treturn _0x1533fb;\n" +
"\t};\n" +
"\tfunction hash(_0x1a5a87) {\n" +
"\t\tvar _0x23a3e0 = {};\n" +
"\t\t_0x23a3e0[_0x1533('0x93', 'Kp2R') + 'J'] = function(_0x5ebeb2, _0x4681fc) {\n" +
"\t\t\treturn _0x5ebeb2 ^ _0x4681fc;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x17', '(bZ%') + 'H'] = function(_0x5108c3, _0x49fe65) {\n" +
"\t\t\treturn _0x5108c3 & _0x49fe65;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x88', 'N#(C') + 'T'] = function(_0x3d3f10, _0x3f2a86) {\n" +
"\t\t\treturn _0x3d3f10 !== _0x3f2a86;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x60', '(cNt') + 'Z'] = _0x1533('0x70', 'K)PG') + 'J';\n" +
"\t\t_0x23a3e0[_0x1533('0x49', 'Y8u@') + 'h'] = function(_0xc3ff68, _0x129ab0) {\n" +
"\t\t\treturn _0xc3ff68 >> _0x129ab0;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x9c', '\$qIi') + 'S'] = function(_0x591dc1, _0x158d9d) {\n" +
"\t\t\treturn _0x591dc1 + _0x158d9d;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x7f', '(bZ%') + 'f'] = function(_0xd6deb, _0x557760) {\n" +
"\t\t\treturn _0xd6deb >> _0x557760;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x4e', 'S9Bn') + 'd'] = function(_0xd05226, _0x29b6e8) {\n" +
"\t\t\treturn _0xd05226 * _0x29b6e8;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x7e', 'y^BN') + 'n'] = function(_0x899f8d, _0x3dbd26) {\n" +
"\t\t\treturn _0x899f8d < _0x3dbd26;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x9a', '(U[3') + 'e'] = function(_0x39678a, _0x37d488) {\n" +
"\t\t\treturn _0x39678a * _0x37d488;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xa3', 'sahq') + 'U'] = function(_0x4cf877, _0x14e1b0) {\n" +
"\t\t\treturn _0x4cf877 << _0x14e1b0;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xa7', 'K#Hh') + 'u'] = function(_0x556336, _0x56b41c) {\n" +
"\t\t\treturn _0x556336 - _0x56b41c;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x3d', '(bZ%') + 'p'] = function(_0x372c20, _0x534fe1) {\n" +
"\t\t\treturn _0x372c20 >> _0x534fe1;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x1d', 'S9Bn') + 'g'] = function(_0x26a0f8, _0x4f1660) {\n" +
"\t\t\treturn _0x26a0f8 >>> _0x4f1660;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0xae', ']*f@') + 'Y'] = function(_0x3423f4, _0x53e1a9) {\n" +
"\t\t\treturn _0x3423f4 | _0x53e1a9;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x26', 's4Dp') + 'P'] = function(_0x19382a, _0x220110) {\n" +
"\t\t\treturn _0x19382a | _0x220110;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x65', 's4Dp') + 'p'] = function(_0x7b95bc, _0x22e933) {\n" +
"\t\t\treturn _0x7b95bc & _0x22e933;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x0', '\$qIi') + 'V'] = function(_0x13269b, _0x227e3b) {\n" +
"\t\t\treturn _0x13269b ^ _0x227e3b;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x29', 'J21I') + 'R'] = function(_0x1ba947, _0x4b40e4) {\n" +
"\t\t\treturn _0x1ba947(_0x4b40e4);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x50', 's4Dp') + 'n'] = _0x1533('0xa', 'qRqa') + 'e';\n" +
"\t\t_0x23a3e0[_0x1533('0x42', 'ksVw') + 'b'] = function(_0x466fa6, _0x3ce8da) {\n" +
"\t\t\treturn _0x466fa6 ^ _0x3ce8da;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x2c', 'rqYb') + 'f'] = function(_0x1e608a, _0x241888) {\n" +
"\t\t\treturn _0x1e608a - _0x241888;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x78', 'J21I') + 'u'] = function(_0x564b78, _0x18d236, _0x8fd805) {\n" +
"\t\t\treturn _0x564b78(_0x18d236, _0x8fd805);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x69', 'zx(Q') + 'D'] = function(_0x3b7666, _0x47f238, _0x57b05e) {\n" +
"\t\t\treturn _0x3b7666(_0x47f238, _0x57b05e);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x5f', 'ZaCV') + 'm'] = function(_0x452fa7, _0x493d76, _0x1446e7, _0x4af814, _0x5e0f59) {\n" +
"\t\t\treturn _0x452fa7(_0x493d76, _0x1446e7, _0x4af814, _0x5e0f59);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x22', 'Rh@#') + 'W'] = function(_0x51a858, _0x440b1c) {\n" +
"\t\t\treturn _0x51a858(_0x440b1c);\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x8', 'xPGU') + 'A'] = function(_0x5e96f2, _0x167b41) {\n" +
"\t\t\treturn _0x5e96f2 + _0x167b41;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x6d', 'rqYb') + 'X'] = function(_0x46cd37, _0x5d8684) {\n" +
"\t\t\treturn _0x46cd37 + _0x5d8684;\n" +
"\t\t};\n" +
"\t\t_0x23a3e0[_0x1533('0x3e', 'rqYb') + 'g'] = function(_0x26bc4b, _0x16bc46) {\n" +
"\t\t\treturn _0x26bc4b(_0x16bc46);\n" +
"\t\t};\n" +
"\t\tvar _0x179134 = _0x23a3e0;\n" +
"\t\tfunction _0x5be8c0(_0x296dd6, _0x778566) {\n" +
"\t\t\treturn _0x179134[_0x1533('0x56', 'ZaCV') + 'J'](_0x179134[_0x1533('0x7c', 'o&Q4') + 'H'](_0x296dd6, 0x7fffffff) + (_0x778566 & 0x7fffffff), _0x179134[_0x1533('0x52', 'K)PG') + 'H'](_0x296dd6, 0x80000000)) ^ _0x778566 & 0x80000000;\n" +
"\t\t}\n" +
"\t\tfunction _0x225a28(_0x2f57af) {\n" +
"\t\t\tvar _0x3c49d8 = {};\n" +
"\t\t\t_0x3c49d8[_0x1533('0x55', 'qRqa') + 'N'] = function(_0x15bfb8, _0x51e1eb) {\n" +
"\t\t\t\treturn _0x15bfb8 < _0x51e1eb;\n" +
"\t\t\t};\n" +
"\t\t\tvar _0x567e0f = _0x3c49d8;\n" +
"\t\t\tvar _0x5b2e48 = _0x1533('0xa4', 'rQf(') + _0x1533('0x4', 'rqYb') + _0x1533('0x95', 'Rh@#') + _0x1533('0x3', 'OhQr');\n" +
"\t\t\tvar _0x321f21 = '';\n" +
"\t\t\tfor (var _0x2a5528 = 0x7; _0x2a5528 >= 0x0; _0x2a5528--) {\n" +
"\t\t\t\tif (_0x179134[_0x1533('0xb2', 'Gfc!') + 'T'](_0x1533('0x2', 'w*Nn') + 'h', _0x179134[_0x1533('0x6c', 'OhQr') + 'Z'])) {\n" +
"\t\t\t\t\t_0x321f21 += _0x5b2e48[_0x1533('0x66', ')T5g') + 'At'](_0x179134[_0x1533('0x9f', 'zx(Q') + 'H'](_0x179134[_0x1533('0x5', 'OhQr') + 'h'](_0x2f57af, _0x2a5528 * 0x4), 0xf));\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\treturn _0x567e0f[_0x1533('0x5a', 'y*wC') + 'N'](t, 0x14) ? 0x5a827999: t < 0x28 ? 0x6ed9eba1: _0x567e0f[_0x1533('0x43', 'zx(Q') + 'N'](t, 0x3c) ? -0x70e44324: -0x359d3e2a;\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\treturn _0x321f21;\n" +
"\t\t}\n" +
"\t\tfunction _0xde630f(_0x10fb9a) {\n" +
"\t\t\tvar _0x5b5b6d = _0x179134[_0x1533('0x16', 'GvA9') + 'S'](_0x179134[_0x1533('0x87', 'N#(C') + 'f'](_0x179134[_0x1533('0x83', '[H[u') + 'S'](_0x10fb9a[_0x1533('0x58', 'OhQr') + 'th'], 0x8), 0x6), 0x1),\n" +
"\t\t\t_0x797793 = new Array(_0x179134[_0x1533('0x59', 'xPGU') + 'd'](_0x5b5b6d, 0x10));\n" +
"\t\t\tfor (var _0x109346 = 0x0; _0x179134[_0x1533('0x61', 'dgE&') + 'n'](_0x109346, _0x179134[_0x1533('0x90', 'y^BN') + 'e'](_0x5b5b6d, 0x10)); _0x109346++) {\n" +
"\t\t\t\tif (_0x179134[_0x1533('0x2a', 'NNf8') + 'T'](_0x1533('0x12', 'sahq') + 'y', _0x1533('0x67', 'y*wC') + 'G')) {\n" +
"\t\t\t\t\t_0x797793[_0x109346] = 0x0;\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\treturn !! [];\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\tfor (_0x109346 = 0x0; _0x109346 < _0x10fb9a[_0x1533('0x40', 'ad*k') + 'th']; _0x109346++) {\n" +
"\t\t\t\t_0x797793[_0x109346 >> 0x2] |= _0x179134[_0x1533('0xa9', '@gLq') + 'U'](_0x10fb9a[_0x1533('0x1c', 'GvA9') + _0x1533('0xf', 'OhQr') + 'At'](_0x109346), _0x179134[_0x1533('0xb3', 'S9Bn') + 'u'](0x18, (_0x109346 & 0x3) * 0x8));\n" +
"\t\t\t}\n" +
"\t\t\t_0x797793[_0x179134[_0x1533('0xb1', 'qRqa') + 'p'](_0x109346, 0x2)] |= 0x80 << 0x18 - (_0x109346 & 0x3) * 0x8;\n" +
"\t\t\t_0x797793[_0x5b5b6d * 0x10 - 0x1] = _0x10fb9a[_0x1533('0xb', '@mqx') + 'th'] * 0x8;\n" +
"\t\t\treturn _0x797793;\n" +
"\t\t}\n" +
"\t\tfunction _0x50e067(_0x45a1cc, _0x3bcee1) {\n" +
"\t\t\treturn _0x45a1cc << _0x3bcee1 | _0x179134[_0x1533('0x5b', 'Y8u@') + 'g'](_0x45a1cc, _0x179134[_0x1533('0x91', '@mqx') + 'u'](0x20, _0x3bcee1));\n" +
"\t\t}\n" +
"\t\tfunction _0x526d3b(_0xb72077, _0xed7f25, _0x5bc269, _0x572b62) {\n" +
"\t\t\tif (_0xb72077 < 0x14) return _0x179134[_0x1533('0x98', 'Gfc!') + 'Y'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x68', 'OhQr') + 'H'](~_0xed7f25, _0x572b62));\n" +
"\t\t\tif (_0x179134[_0x1533('0x9e', 'w*Nn') + 'n'](_0xb72077, 0x28)) return _0x179134[_0x1533('0x99', ')T5g') + 'J'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
"\t\t\tif (_0xb72077 < 0x3c) return _0x179134[_0x1533('0x13', 'CNzQ') + 'P'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x45', '@mqx') + 'p'](_0xed7f25, _0x572b62)) | _0x179134[_0x1533('0xa0', 'avK7') + 'p'](_0x5bc269, _0x572b62);\n" +
"\t\t\treturn _0x179134[_0x1533('0x5e', 'avK7') + 'V'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
"\t\t}\n" +
"\t\tfunction _0x496785(_0x583c97) {\n" +
"\t\t\treturn _0x583c97 < 0x14 ? 0x5a827999: _0x583c97 < 0x28 ? 0x6ed9eba1: _0x583c97 < 0x3c ? -0x70e44324: -0x359d3e2a;\n" +
"\t\t}\n" +
"\t\tvar _0x45b840 = _0x179134[_0x1533('0x39', 'R#BF') + 'R'](_0xde630f, _0x1a5a87);\n" +
"\t\tvar _0x53258d = new Array(0x50);\n" +
"\t\tvar _0xc86ce6 = 0x67452301;\n" +
"\t\tvar _0x3ee50e = -0x10325477;\n" +
"\t\tvar _0xdb1008 = -0x67452302;\n" +
"\t\tvar _0xe99bcb = 0x10325476;\n" +
"\t\tvar _0x56c860 = -0x3c2d1e10;\n" +
"\t\tfor (var _0x4c314c = 0x0; _0x4c314c < _0x45b840[_0x1533('0x46', '\$qIi') + 'th']; _0x4c314c += 0x10) {\n" +
"\t\t\tvar _0x41e8e8 = _0xc86ce6;\n" +
"\t\t\tvar _0x1b7277 = _0x3ee50e;\n" +
"\t\t\tvar _0x30f7c3 = _0xdb1008;\n" +
"\t\t\tvar _0x279c95 = _0xe99bcb;\n" +
"\t\t\tvar _0xf6e888 = _0x56c860;\n" +
"\t\t\tfor (var _0x77e31b = 0x0; _0x179134[_0x1533('0x34', 'Y8u@') + 'n'](_0x77e31b, 0x50); _0x77e31b++) {\n" +
"\t\t\t\tif (_0x77e31b < 0x10) {\n" +
"\t\t\t\t\tif (_0x179134[_0x1533('0xa6', 'Mifp') + 'T'](_0x1533('0x3f', 'o&Q4') + 'G', _0x179134[_0x1533('0xa1', 'rqYb') + 'n'])) {\n" +
"\t\t\t\t\t\t_0x53258d[_0x77e31b] = _0x45b840[_0x4c314c + _0x77e31b];\n" +
"\t\t\t\t\t} else {\n" +
"\t\t\t\t\t\tsleepT = _0x179134[_0x1533('0x48', 'NNf8') + 'R'](parseInt, data['wt']) > ret[0x1] ? _0x179134[_0x1533('0x97', 'd[i)') + 'R'](parseInt, data['wt']) - ret[0x1] : 0x1f4;\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t} else {\n" +
"\t\t\t\t\t_0x53258d[_0x77e31b] = _0x50e067(_0x179134[_0x1533('0x96', 'rQf(') + 'b'](_0x179134[_0x1533('0xb0', 'Gfc!') + 'b'](_0x53258d[_0x179134[_0x1533('0xa5', 'K)PG') + 'u'](_0x77e31b, 0x3)], _0x53258d[_0x179134[_0x1533('0x51', 'K)PG') + 'f'](_0x77e31b, 0x8)]) ^ _0x53258d[_0x77e31b - 0xe], _0x53258d[_0x179134[_0x1533('0xaf', '(U[3') + 'f'](_0x77e31b, 0x10)]), 0x1);\n" +
"\t\t\t\t}\n" +
"\t\t\t\tt = _0x179134[_0x1533('0x18', 'GvA9') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x23', '\$qIi') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x8d', 'Mifp') + 'D'](_0x50e067, _0xc86ce6, 0x5), _0x179134[_0x1533('0x4c', 'xPGU') + 'm'](_0x526d3b, _0x77e31b, _0x3ee50e, _0xdb1008, _0xe99bcb)), _0x5be8c0(_0x5be8c0(_0x56c860, _0x53258d[_0x77e31b]), _0x179134[_0x1533('0xad', 'rqYb') + 'W'](_0x496785, _0x77e31b)));\n" +
"\t\t\t\t_0x56c860 = _0xe99bcb;\n" +
"\t\t\t\t_0xe99bcb = _0xdb1008;\n" +
"\t\t\t\t_0xdb1008 = _0x50e067(_0x3ee50e, 0x1e);\n" +
"\t\t\t\t_0x3ee50e = _0xc86ce6;\n" +
"\t\t\t\t_0xc86ce6 = t;\n" +
"\t\t\t}\n" +
"\t\t\t_0xc86ce6 = _0x5be8c0(_0xc86ce6, _0x41e8e8);\n" +
"\t\t\t_0x3ee50e = _0x5be8c0(_0x3ee50e, _0x1b7277);\n" +
"\t\t\t_0xdb1008 = _0x5be8c0(_0xdb1008, _0x30f7c3);\n" +
"\t\t\t_0xe99bcb = _0x5be8c0(_0xe99bcb, _0x279c95);\n" +
"\t\t\t_0x56c860 = _0x5be8c0(_0x56c860, _0xf6e888);\n" +
"\t\t}\n" +
"\t\treturn _0x179134[_0x1533('0x86', 'Gfc!') + 'A'](_0x179134[_0x1533('0x44', 'DDSy') + 'X'](_0x179134[_0x1533('0x8e', 'xPGU') + 'g'](_0x225a28, _0xc86ce6), _0x179134[_0x1533('0x77', 'n*Rs') + 'g'](_0x225a28, _0x3ee50e)) + _0x225a28(_0xdb1008) + _0x225a28(_0xe99bcb), _0x179134[_0x1533('0xd', 'DDSy') + 'g'](_0x225a28, _0x56c860));\n" +
"\t}\n" +
"\tfunction go(_0x394549) {\n" +
"\t\tvar _0x33d0d6 = {};\n" +
"\t\t_0x33d0d6[_0x1533('0x2b', 'sahq') + 'H'] = _0x1533('0x14', 'rQf(') + _0x1533('0x15', 'sahq');\n" +
"\t\t_0x33d0d6[_0x1533('0xab', 'dgE&') + 'H'] = function(_0x23f209, _0x4ed415) {\n" +
"\t\t\treturn _0x23f209 < _0x4ed415;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x5c', 'xPGU') + 'g'] = function(_0x39bd0a, _0x4d5764) {\n" +
"\t\t\treturn _0x39bd0a != _0x4d5764;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x37', 'y*wC') + 'N'] = function(_0x3c9a26, _0x109b89) {\n" +
"\t\t\treturn _0x3c9a26 + _0x109b89;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x63', 'gkl*') + 'z'] = function(_0x584935, _0x58347c) {\n" +
"\t\t\treturn _0x584935 + _0x58347c;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x21', 'ZaCV') + 'z'] = function(_0x3fc795, _0x6962f6) {\n" +
"\t\t\treturn _0x3fc795 + _0x6962f6;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x27', '(cNt') + 'I'] = function(_0x303446, _0x3388fb) {\n" +
"\t\t\treturn _0x303446 == _0x3388fb;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x6b', 'y*wC') + 't'] = function(_0x5cb515, _0x31e7f3) {\n" +
"\t\t\treturn _0x5cb515(_0x31e7f3);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x75', 'Rh@#') + 'E'] = function(_0x529d8e, _0xc1423a) {\n" +
"\t\t\treturn _0x529d8e + _0xc1423a;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x62', 'rQf(') + 'a'] = function(_0x453505, _0x50cd56) {\n" +
"\t\t\treturn _0x453505 + _0x50cd56;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x2e', 'NNf8') + 'B'] = _0x1533('0x28', ']*f@') + _0x1533('0x57', 'Kp2R') + '\\x20/';\n" +
"\t\t_0x33d0d6[_0x1533('0xe', 'Y8u@') + 'y'] = function(_0x25bcfd, _0x4b634e, _0x1bef22) {\n" +
"\t\t\treturn _0x25bcfd(_0x4b634e, _0x1bef22);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x92', 'J21I') + 'i'] = function(_0xccfc4d, _0x9a8620) {\n" +
"\t\t\treturn _0xccfc4d > _0x9a8620;\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x11', '@H8e') + 'n'] = function(_0x1b9fc7, _0x7d3ce) {\n" +
"\t\t\treturn _0x1b9fc7(_0x7d3ce);\n" +
"\t\t};\n" +
"\t\t_0x33d0d6[_0x1533('0x1b', 'N#(C') + 'O'] = _0x1533('0x7', 'N#(C') + '失败';\n" +
"\t\tvar _0x2eda7d = _0x33d0d6;\n" +
"\t\tfunction _0x29e502() {\n" +
"\t\t\tvar _0x24a3c1 = window[_0x1533('0x35', 'ad*k') + _0x1533('0x5d', 'Y8u@') + 'r'][_0x1533('0x19', 'avK7') + _0x1533('0x7d', 'cJ3U') + 't'],\n" +
"\t\t\t_0x5f1192 = [_0x2eda7d[_0x1533('0x24', '(bZ%') + 'H']];\n" +
"\t\t\tfor (var _0x2d8686 = 0x0; _0x2eda7d[_0x1533('0x1a', 'avK7') + 'H'](_0x2d8686, _0x5f1192[_0x1533('0x3b', 'Y8u@') + 'th']); _0x2d8686++) {\n" +
"\t\t\t\tif (_0x2eda7d[_0x1533('0x2f', 'y*wC') + 'g'](_0x24a3c1[_0x1533('0x76', 'y*wC') + _0x1533('0x33', 'ZaCV')](_0x5f1192[_0x2d8686]), -0x1)) {\n" +
"\t\t\t\t\treturn !! [];\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t\tif (window[_0x1533('0x41', 'dgE&') + _0x1533('0x6e', 'dgE&') + _0x1533('0x85', 'NNf8')] || window[_0x1533('0x30', 'o&Q4') + _0x1533('0x38', 'OhQr')] || window[_0x1533('0xc', 'OhQr') + _0x1533('0x74', 'qRqa')] || window[_0x1533('0x25', '(bZ%') + _0x1533('0x4f', 'avK7') + 'r'][_0x1533('0xa2', 'xPGU') + _0x1533('0x53', ')T5g') + 'r'] || window[_0x1533('0xaa', 'cJ3U') + _0x1533('0x6', 'K#@S') + 'r'][_0x1533('0x8f', 'rqYb') + _0x1533('0xac', '(bZ%') + _0x1533('0x9', 'Rh@#') + _0x1533('0x4d', 'rQf(') + 'e'] || window[_0x1533('0x31', 'N#(C') + _0x1533('0x79', 'dgE&') + 'r'][_0x1533('0x4b', 'ksVw') + _0x1533('0x10', 'gkl*') + _0x1533('0xa8', 'd[i)') + _0x1533('0x89', '\$qIi') + _0x1533('0x84', 's4Dp')]) {\n" +
"\t\t\t\treturn !! [];\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\tif (_0x29e502()) {\n" +
"\t\t\treturn;\n" +
"\t\t}\n" +
"\t\tvar _0x1642ef = new Date();\n" +
"\t\tfunction _0x1a1fd9(_0x55baea, _0x579edb) {\n" +
"\t\t\tvar _0x551ac6 = _0x394549[_0x1533('0x4a', 'y*wC') + 's'][_0x1533('0x71', 'K#@S') + 'th'];\n" +
"\t\t\tfor (var _0x54ccec = 0x0; _0x2eda7d[_0x1533('0x6f', 'dFiI') + 'H'](_0x54ccec, _0x551ac6); _0x54ccec++) {\n" +
"\t\t\t\tfor (var _0x562c2d = 0x0; _0x2eda7d[_0x1533('0x9b', '[H[u') + 'H'](_0x562c2d, _0x551ac6); _0x562c2d++) {\n" +
"\t\t\t\t\tvar _0x1b8d8c = _0x2eda7d[_0x1533('0x80', '[H[u') + 'N'](_0x2eda7d[_0x1533('0x3a', 'w*Nn') + 'z'](_0x2eda7d[_0x1533('0x8b', 'cJ3U') + 'z'](_0x579edb[0x0], _0x394549[_0x1533('0x32', 'R#BF') + 's'][_0x1533('0x82', '@mqx') + 'tr'](_0x54ccec, 0x1)), _0x394549[_0x1533('0x1f', '*hmq') + 's'][_0x1533('0x47', '(cNt') + 'tr'](_0x562c2d, 0x1)), _0x579edb[0x1]);\n" +
"\t\t\t\t\tif (_0x2eda7d[_0x1533('0x6a', '(bZ%') + 'I'](_0x2eda7d[_0x1533('0x9d', 'Kp2R') + 't'](hash, _0x1b8d8c), _0x55baea)) {\n" +
"\t\t\t\t\t\treturn [_0x1b8d8c, new Date() - _0x1642ef];\n" +
"\t\t\t\t\t}\n" +
"\t\t\t\t}\n" +
"\t\t\t}\n" +
"\t\t};\n" +
"\t\tvar _0x1e73b9 = _0x2eda7d[_0x1533('0x36', 'sahq') + 'y'](_0x1a1fd9, _0x394549['ct'], _0x394549[_0x1533('0x2d', '\$qIi')]);\n" +
"\t\tif (_0x1e73b9) {\n" +
"\t\t\tvar _0x3b1427;\n" +
"\t\t\tif (_0x394549['wt']) {\n" +
"\t\t\t\t_0x3b1427 = _0x2eda7d[_0x1533('0x1', 'zx(Q') + 'i'](parseInt(_0x394549['wt']), _0x1e73b9[0x1]) ? _0x2eda7d[_0x1533('0x54', 'avK7') + 'n'](parseInt, _0x394549['wt']) - _0x1e73b9[0x1] : 0x1f4;\n" +
"\t\t\t} else {\n" +
"\t\t\t\t_0x3b1427 = 0x5dc;\n" +
"\t\t\t}\n" +
"\t\t\t_0x2eda7d[_0x1533('0x72', 'qRqa') + 'y'](setTimeout,\n" +
"\t\t\tfunction() {\n" +
"\t\t\t\tdocument[_0x1533('0x20', 'rQf(') + 'ie'] = _0x2eda7d[_0x1533('0x73', 'rQf(') + 'E'](_0x2eda7d[_0x1533('0x3c', 'o&Q4') + 'a'](_0x394549['tn'] + '=' + _0x1e73b9[0x0] + (_0x1533('0x8a', 'ksVw') + _0x1533('0x1e', 'GvA9') + '='), _0x394549['vt']), _0x2eda7d[_0x1533('0x8c', 'Gfc!') + 'B']);\n" +
"\t\t\t\tlocation[_0x1533('0x7a', 'sahq')] = location[_0x1533('0x64', 'd[i)') + _0x1533('0x81', '(U[3')] + location[_0x1533('0x94', 'sahq') + 'ch'];\n" +
"\t\t\t},\n" +
"\t\t\t_0x3b1427);\n" +
"\t\t} else {\n" +
"\t\t\talert(_0x2eda7d[_0x1533('0x7b', ')T5g') + 'O']);\n" +
"\t\t}\n" +
"\t};\n" +
"\tgo({\n" +
"\t\t\"bts\": [\"1618915727.945|0|HM5\", \"VoVNIPqcrvFO13woIZK2RY%3D\"],\n" +
"\t\t\"chars\": \"ZDUZR0cDjTEXznMkfgavFO\",\n" +
"\t\t\"ct\": \"3049e28877124ad21ca0db55e3af95ff316ceef6\",\n" +
"\t\t\"ha\": \"sha1\",\n" +
"\t\t\"tn\": \"__jsl_clearance\",\n" +
"\t\t\"vt\": \"3600\",\n" +
"\t\t\"wt\": \"1500\"\n" +
"\t})"
ScriptEngineManager manager = new ScriptEngineManager();
ScriptEngine engine = manager.getEngineByName("javascript");
engine.eval(str);
//获取对象
System.out.println();
}
def getJsContent(def page){
ScriptEngineManager manager = new ScriptEngineManager();
//得到脚本引擎
ScriptEngine engine = manager.getEngineByName("javascript");
//处理加密js
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
try {
//得到解密后的js
js = js.replace("document.cookie","var cookie")
js = js.replace("location.href=location.pathname+location.search", "")
engine.eval(js);
def v = engine.get("cookie")
return v.toString().substring(0, v.toString().indexOf(";"))
} catch (ScriptException e) {
e.printStackTrace();
}
return null
}
def getJsContent2(def page){
ScriptEngineManager manager = new ScriptEngineManager();
//得到脚本引擎
ScriptEngine engine = manager.getEngineByName("javascript");
//处理加密js
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
try {
//得到解密后的js
//得到解密后的js
String result = (String) engine.eval(js);
result = result.substring(result.indexOf("var cd"),result.indexOf("dc+=cd;")+7);
result = result.replaceAll("document*.*toLowerCase\\(\\)", "'x'");
//得到cookie
String jsl = (String) engine.eval(result);
return jsl;
} catch (ScriptException e) {
e.printStackTrace();
}
return null
}
/**
* 通过破解动态JavaScript脚本,
* 获取cookie名为 __jsl_clearance的值
*
* @param body 相应内容(一般为第一次请求获取到的动态js字符串)
* @return cookie名为 __jsl_clearance的值
*/
private static String getJslClearance(String body) {
//V8:谷歌开源的运行JavaScript脚本的库. 参数:globalAlias=window, 表示window为全局别名,
// 告诉V8在运行JavaScript代码时, 不要从代码里找window的定义.
V8 runtime = V8.createV8Runtime("window");
//将第一次请求pdf资源时获取到的字符串提取成V8可执行的JavaScript代码
body = body.trim()
.replace("<script>", "")
.replace("</script>", "")
.replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))",
"y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})");
//用V8执行该段代码获取新的动态JavaScript脚本
String result = runtime.executeStringScript(body);
//获取 jsl_clearance 的第一段, 格式形如: 1543915851.312|0|
String startStr = "document.cookie='";
int i1 = result.indexOf(startStr) + startStr.length();
int i2 = result.indexOf("|0|");
String cookie1 = result.substring(i1, i2 + 3);
/*
获取 jsl_clearance 的第二段,格式形如: DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D。
主要原理是: 新的动态JavaScript脚本是为浏览器设置cookie, 且cookie名为__jsl_clearance
其中第一段值(格式形如:1543915851.312|0|)已经明文写好, 用字符串处理方法即可获取.
第二段则是一段JavaScript函数, 需要有V8运行返回,
该函数代码需要通过一些字符串定位, 提取出来, 交给V8运行.
*/
startStr = "|0|'+(function(){";
int i3 = result.indexOf(startStr) + startStr.length();
int i4 = result.indexOf("})()+';Expires");
String code = result.substring(i3, i4).replace(";return", ";");
String cookie2 = runtime.executeStringScript(code);
/*
拼接两段字符串, 返回jsl_clearance的完整的值.
格式形如: 1543915851.312|0|DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D
*/
return cookie1 + cookie2;
}
/**
* 将HTTP响应体转换为字符串返回
*
* @param response HTTP响应
* @return 响应体的字符串形式
* @throws IOException IO异常
*/
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
}
static String getJsluid(HttpResponse response) {
Header header = response.getFirstHeader("set-cookie");
String[] split = header.getValue().split(";");
for (String s : split) {
if (s.contains("__jsluid")) {
return s.trim();
}
}
return "";
}
def fristCrawl(){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Cache-Control","max-age=0")
// headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Accept-Encoding","gzip, deflate")
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
void process(Object args) {
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
for(def key : companyKeys){
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
def s = crawl(url, queryBody);
if(s){
print s
parser(s);
}
println s
}
}
def parser(content){
def document = Jsoup.parse(content);
if(document){
def detailUrls = []
Elements elements = JQueryUtils.select([".search_list_item"], document)
if(elements && elements.size()>0){
for(def el : elements){
detailUrls << el.attr("href")
}
}
}
}
def static crawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","www.gsxt.gov.cn")
headers.put("Connection","keep-alive")
headers.put("Content-Length","226")
headers.put("Cache-Control","max-age=0")
headers.put("Upgrade-Insecure-Requests","1")
headers.put("Origin","http://www.gsxt.gov.cn")
headers.put("Content-Type","application/x-www-form-urlencoded")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
// headers.put("Accept-Encoding","gzip, deflate")
headers.put("Accept-Language","zh-CN,zh;q=0.9")
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
checkAbuyun(true);
def responseMap = [:]
// 创建代理服务器地址对象
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
// 创建HTTP类型代理对象
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
if(content){
outputStream.write(content.getBytes());
}
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
public static checkAbuyun(boolean check) {
if (check) {
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
e.printStackTrace()
}
}
AbuyunRequestCount.incrementAndGet();
} else {
AbuyunRequestCount.decrementAndGet();
}
}
}
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
districtMap.putAll(detailMap)
}
}
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
while(flag){
println "开始加载解析数据......"
@@ -116,7 +116,7 @@ class DealPriceMathBatch extends BaseBatch{
count++;
}catch (Exception e){
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.getMessage()
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.printStackTrace()
}finally {
latch.countDown()
}
@@ -442,16 +442,21 @@ class DealPriceMathBatch extends BaseBatch{
if(regionMap){
regionMap.each{k,v->
def arrays = k.split("###")
if(arrays[1] && arrays[1]==keyword){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
try{
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
}
}catch(Exception e){
e.printStackTrace()
}
}
}
}else{//无区域
cityMap.each{r,dMap->
dMap.each{k,v->
def arrays = k.split("###")
if(arrays[1] && arrays[1]==keyword){
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
}
}
@@ -596,10 +601,13 @@ class DealPriceMathBatch extends BaseBatch{
}
def getOpenApiMatch(def city, def region, def name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
if(name){
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
if(jsonObject && jsonObject.credit>0 ){
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
}
}
return null;
}
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
houseLoudng.fitment_cost = ret.fitment_charge
houseLoudng.car_port = ret.carport
houseLoudng.house_height = ret.house_hight
houseLoudng.worry_facility" : null,
houseLoudng.fitment_age = ret.fitment_age
houseLoudng.lighting_ventilation" : null,
houseLoudng.build_type = ret.build_type
houseLoudng.house_property" : null,
houseLoudng.noise" : null,
houseLoudng.sight" : null,
houseLoudng.fitment_rank" : null,
houseLoudng.sideset" : null,
// houseLoudng.worry_facility" : null,
// houseLoudng.fitment_age = ret.fitment_age
// houseLoudng.lighting_ventilation" : null,
// houseLoudng.build_type = ret.build_type
// houseLoudng.house_property" : null,
// houseLoudng.noise" : null,
// houseLoudng.sight" : null,
// houseLoudng.fitment_rank" : null,
// houseLoudng.sideset" : null,
houseLoudng.district_no = district._id
@@ -352,7 +352,7 @@ class HouseExportBatch extends BaseBatch{
def rets = []
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-2020*", queryBuilder);
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "latest_seawater", queryBuilder);
if(houses){
for (int k=0;k<houses.size();k++) {
def map = houses[k];
File diff suppressed because it is too large Load Diff
@@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream
@@ -36,16 +37,17 @@ class GaoDeMapCrawl {
public static final def dbUrl = "mongodb://192.168.0.235:27018";
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
final AtomicInteger count = new AtomicInteger()
def updateLists = []
def updateErrorLists = []
def flagLists = []
def index = 0
def prefix = "road_"
def prefix = "district_"
// def categorys = ["141202", "141203"]//中学,小学
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
def categorys = ["190301"]//道路相关
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
// def categorys = ["190301"]//道路相关
def totalCount = new AtomicLong()
static main(args) {
def task = new GaoDeMapCrawl();
@@ -109,26 +111,13 @@ class GaoDeMapCrawl {
def run(){//,"西安","重庆"
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"]
def citys = CommonUtils.findAllCity()
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
// def citys = CommonUtils.findAllCity()
def totalCount = 0;
for(def city : citys){
def cityLations = findAllLation(city);
println "开始处理["+city +"]:"+cityLations.size()
gaoDeCitySearch(cityLations);
if(updateLists.size() > 0){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
}
@@ -145,39 +134,68 @@ class GaoDeMapCrawl {
}
def gaoDeCitySearch(def cityLocations){
try {
for(def location : cityLocations){
index ++
// def bound_gd = location.bound_gd
// final CountDownLatch latch = new CountDownLatch(cityLocations.size())
for(def i=0;i<cityLocations.size();i++){
def location = cityLocations[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
// try{
// totalCount.addAndGet(1)
searching(location)
// }catch(Exception e){
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
// }
// });
}
// latch.await()
if(updateLists.size() > 0){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
def searching(location){
// def bound_gd = location.bound_gd
// def lats = [bound_gd[1], bound_gd[3]]
// def lngs = [bound_gd[0], bound_gd[2]]
// lats.sort()
// lngs.sort()
// println index
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
search(location)
search(location)
// }
if(updateLists.size() > 0 && updateLists.size()>1000){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0 && flagLists.size()>1000){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
}catch (Exception e){
e.printStackTrace()
if(updateLists.size() > 0 && updateLists.size()>1000){
println " 正常访问共:"+updateLists.size()
updateBatch();
}
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
println " 失败访问共:"+updateErrorLists.size()
updateErrorBatch();
}
if(flagLists.size()>0 && flagLists.size()>1000){
println "有数据的格子共:"+flagLists.size()
updateFlagBath();
}
}
def search(location){
def bound_gd = location.bound_gd
def lats = [bound_gd[1], bound_gd[3]]
@@ -198,9 +216,11 @@ class GaoDeMapCrawl {
if(m.rets){
processData(m.rets);
location.count = m.rets.size()
location.content = m.toString()
flagLists.add(location)
}else{
location.count = 0
location.content = m.toString()
flagLists.add(location)
}
@@ -210,7 +230,7 @@ class GaoDeMapCrawl {
updateErrorLists.addAll(location)
}
// update(location);
println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
}
def processData(rets){
@@ -336,7 +356,7 @@ class GaoDeMapCrawl {
for(def location : updateLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","完成").append("m_date", new Date())
def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -358,7 +378,7 @@ class GaoDeMapCrawl {
for(def location : flagLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date())
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -382,7 +402,7 @@ class GaoDeMapCrawl {
for(def location : updateErrorLists){
if(location && location._id){
Document updateDocument = new Document();
def doc = new Document(prefix+"status","失败").append("m_date", new Date())
def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content)
updateDocument.append('$set', doc);
Bson updateBson = Filters.eq("_id", location._id);
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
@@ -0,0 +1,301 @@
package com.fangjia.script.gj
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import org.apache.lucene.queryparser.classic.QueryParser
class LogStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
static main(args) {
def task = new LogStat()
task.run()
// task.run1()
}
def run1(){
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d ->
def city = d['C'].replace("市","")
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
if(!district){
println "sss"
}
if(d['AT'] && district){
def startDate2 = DateUtil.add(d['X'], -365)
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
def price1 = d['AI']
def price2 = d['AL']
def price3 = d['AQ']
def startPrice = 0
def endPrice = 0
def lastPrice = 0
if(price1){
startPrice = (price1 - (price1*0.15)).toInteger()
endPrice = (price1 + (price1*0.15)).toInteger()
lastPrice = price1
}else if(price2){
startPrice = (price2 - (price2*0.15)).toInteger()
endPrice = (price2 + (price2*0.15)).toInteger()
lastPrice = price2
}else if(price3){
startPrice = (price3 - (price3*0.15)).toInteger()
endPrice = (price3 + (price3*0.15)).toInteger()
lastPrice = price3
}
def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
if(prices && prices.size()>=3){
def houseAvgPrice = (prices.sum()/prices.size()).toInteger()
d['AD'] = houseAvgPrice
d['AC'] = (houseAvgPrice * d['N']).toInteger()
}else{
def random = new Random().nextInt()%6
def randomAvgPrice = lastPrice*random/100 + lastPrice
d['AD'] = randomAvgPrice
d['AC'] = (randomAvgPrice * d['N']).toInteger()
}
}
},0);
}
def run(){
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d ->
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
if(d['AA'] && d['AB'] && d['AH']){
def city = d['C'].replace("市","")
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
if(district){
def alias = []
if(district.alias){
if(!district.alias.contains(district.name)){
district.alias << district.name
}
alias = district.alias
}else{
alias << district.name
}
// def startDate = DateUtil.add(d['X'], -90)
// def endDate = DateUtil.add(d['X'], 5)
// def trades = findTrades(district, startDate, endDate)
// if(trades){
// d['AJ'] = (int)(trades.sum()/trades.size())
// d['AK'] = trades.size()
// }
//
// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X'])
// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1)))
// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger()
// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger()
// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd)
// if(prices){
// d['AL'] = (int)(prices.sum()/prices.size())
// d['AM'] = prices.size()
// }
//
//
// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger()
// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price")
// if(dbo){
// d['AN'] = dbo.lianjia_esf_price
// }
def price = d['AH']
def startPrice = (price - (price*0.15)).toInteger()
def endPrice = (price + (price*0.15)).toInteger()
def startDate2 = DateUtil.add(d['X'], -365)
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
if(count > 0){
d['AS'] = count
}
}
}
},0)
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
q.put("source", "链家在线")
def alias = []
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
alias = dist.alias
}else{
alias << dist.name
}
if(alias){
q.put("district_name", new BasicDBObject('$in', alias))
}
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def prices = []
for(def d : dbcorsor){
if(d.avg_price){
prices << d.avg_price
}
}
return prices
}
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
def prices = []
ElasticUtils.executeSqlQuery(sql,{data->
prices << data.avg_price
})
return prices
}
def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
def total = 0
ElasticUtils.executeSqlQuery(sql,{data->
total = data.c
})
return total
}
def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
def aliasStr = escapeExprSpecialWord(alias.join("','"))
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
def prices = []
ElasticUtils.executeSqlQuery(sql,{data->
prices << data.avg_price
})
return prices
}
/**
* 转义字符串中的特殊字符
* 仅过滤Lucene regexps, 其他的勿用
* @param keyword
* @return
*/
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
// def findTestDatas(def city, def value) {
// def q = new BasicDBObject()
// q.put("user_id", "1007751")
// q.put("parameter_map.city", city)
// q.put("request_uri", "/property/evaluateForTest")
//
// def keys = new BasicDBObject()
// keys.put("res_map",1)
// def cursor = LOG.find(q)
// def count = 0
// println "总量:"+cursor.count()
// def datas = []
// for(def d : cursor){
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
// def resMap = d.res_map.result
// if(resMap.calcType==value){
// def data = []
// def rowMap = [:]
// rowMap.city = resMap.city
// rowMap.region = resMap.district
// rowMap.name = resMap.name
// rowMap.calDate = d.parameter_map.calDate
// data << rowMap
// }
// }
// }
// return datas;
// }
// for(def i=0;i<datas.size();i++){
// def d = datas[i];
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
// if(dbo){
// d['E'] = dbo.avg_price
// }
//
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
// if(district){
// def startDate = DateUtil.addMonth(calDate, -6)
// def start = startDate.replace("-","").toInteger()
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
// def alias = []
// if(district.alias){
// if(!district.alias.contains(district.name)){
// district.alias << district.name
// }
// alias = district.alias
// }else{
// alias << district.name
// }
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
// d['G'] = count
// }
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", name)
q.put("s_date", s_date)
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
def obj = null;
for(def d : dbcursor){
obj = d;
break;
}
return obj;
}
}
@@ -47,7 +47,7 @@ class YouDaDataParser {
def final AtomicInteger count_ = new AtomicInteger(0)
static def statDate = "2021-02-01"
static def statDate = "2021-03-01"
static main(args) {
def task = new YouDaDataParser();
// task.newHouseparse();//友达一手房存入mongo
@@ -285,7 +285,7 @@ class YouDaDataParser {
def esfParser(){
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
def datas = []
for(def fileName : files){
def rows = readDBF(fileName);
@@ -0,0 +1,175 @@
package com.fangjia.script.gj.address
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.JSONObject
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class AddressParser {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddressParser();
task.run()
// task.gaoDeAddress()
}
def gaoDeAddress(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match2(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def run(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
final AtomicInteger count = new AtomicInteger()
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
count.addAndGet(1)
println count.get() +" "+data.district_name
match(data._id, data.city, data.region, data.school_address, data.school_name);
} catch (Exception e) {
println "异常"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def match(id, city, region, address, name){
def locationMap = null;
// if(region && !name.contains(region)){
// locationMap = matchLocation(city, region + name, "name")
// }else{
locationMap = matchLocation(city + region, name, "name")
// }
// if(!locationMap){
// locationMap = matchLocation(city, address, "address")
// }
if(locationMap){
DBObject data = new BasicDBObject()
data.put("school_lat1", locationMap.glat)
data.put("school_lng1", locationMap.glng)
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def match2(id, city, region, address, name){
def locationMap = null;
locationMap = matchLocation2(city + region, name, "name")
if(!locationMap){
locationMap = matchLocation2(city, address, "address")
}
if(locationMap){
DBObject data = new BasicDBObject()
data.put("gaode_name", locationMap.name)
data.put("gaode_address", locationMap.address)
if(locationMap.location){
def arrs = locationMap.location.split(",")
data.put("school_lat", arrs[1].toDouble())
data.put("school_lng", arrs[0].toDouble())
}
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
}
}
def matchLocation(city, address, cat){
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
def map = [:]
if (gloJson && gloJson.get("geocodes")) {
def geocodes = gloJson.get("geocodes")
def datas = geocodes.get(0)
if(datas.formatted_address && cat=="name"){
if(datas.formatted_address.contains(address)){
def glocation = datas.location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}else{
return null
}
}else{
def glocation = geocodes.get(0).location
def los = glocation.split(",")
map.glng = Double.parseDouble(los[0].toString())
map.glat = Double.parseDouble(los[1].toString())
}
}
return map
}
def matchLocation2(city, address, cat){
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
def map = [:]
if (gloJson && gloJson.get("pois")) {
def geocodes = gloJson.get("pois")
def datas = geocodes.get(0)
return datas
}
return null
}
def findAllDatas(){
def q = new BasicDBObject()
// q.put("school_name","苏州市娄葑实验小学")
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
}
@@ -0,0 +1,354 @@
package com.fangjia.script.gj.ccb
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Pattern
class DistrictSchoolDataExport {
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def regionCodeMap = [:]
static main(args) {
def task = new DistrictSchoolDataExport()
// task.run()
task.megre()
}
def megre(){
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
def ids = []
for(def d : datas1){
ids << d['U']
}
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
for(def d : datas2){
def id = d['U']
if(!ids.contains(id)){
datas1 << d
}
}
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
}
def run(){
def city = "苏州"
getRegionCodeByCity(city)
getRegionCodeByCity("合肥")
getRegionCodeByCity("西安")
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
def districts = []
districts.addAll(districts0)
districts.addAll(districts1)
districts.addAll(districts2)
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def rets = []
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
def i = 0
for(def k=0;k<districts.size();k++){
final def dist = districts.get(k)
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
if(primarySchools || midSchools){
def info1 = null;
def info2 = null;
def distance1 = 0
def distance2 = 0
if(primarySchools && dist.lng){
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
if(info1 && info1.school_lat2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
if(distance1>3000){
info1 = null
}
}else{
info1 = null
}
}
if(midSchools) {
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
if(info2 && info2.school_lng2){
double lat = dist.lat/10000000000.0
double lng = dist.lng/10000000000.0
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
if(distance2>3000){
info2 = null
}
}else{
info2 = null
}
}
if(info1|| info2){
def row = []
row << cityCodeMap.get(dist.city)
row << regionCodeMap.get(dist.city +"-"+dist.region)
row << dist.name
row << dist.lng/10000000000.0
row << dist.lat/10000000000.0
row << dist.address
if(info1){
row << info1.school_name
row << info1.school_address!=null?info1.school_address:''
row << info1.attribute!=null?info1.attribute:''
row << info1.school_lng2!=null?info1.school_lng2:''
row << info1.school_lat2!=null?info1.school_lat2:''
// row << distance1
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
if(info2){
row << info2.school_name
row << info2.school_address!=null?info2.school_address:''
row << info2.attribute!=null?info2.attribute:''
row << info2.school_lng2!=null?info2.school_lng2:''
row << info2.school_lat2!=null?info2.school_lat2:''
// row << distance2
}else{
row << ""
row << ""
row << ""
row << ""
row << ""
// row << ""
}
row << ""
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
row << "fjw"
row << dist._id.toString()
row << "对口"
rets << row
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
};
})
}
latch.await()
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
}
def selectSchool(lng, lat, def schools){
def minSchool = null
def level0 = []//国家
def level1 = []//省重点
def level2 = []//市重点
def level3 = []//区重点
def level4 = []//普通
def level5 = []//其它
for(def s : schools){
if(s.attribute){
if(s.attribute.contains("国家重点")){
level0 << s
}else if(s.attribute.contains("省重点")){
level1 << s
}else if(s.attribute.contains("市重点")){
level2 << s
}else if(s.attribute.contains("区重点")){
level3 << s
}else if(s.attribute.contains("普通")){
level4 << s
}else{
level5 << s
}
}else{
level5 << s
}
}
if(level0){
if(level0.size()==1){
minSchool = level0.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level0)
}
}else if(level1){
if(level1.size()==1){
minSchool = level1.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level1)
}
}else if(level2){
if(level2.size()==1){
minSchool = level2.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level2)
}
}else if(level3){
if(level3.size()==1){
minSchool = level3.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level3)
}
}else if(level4){
if(level4.size()==1){
minSchool = level4.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level4)
}
}else if(level5){
if(level5.size()==1){
minSchool = level5.get(0)
}else{
minSchool = selectMinDistanceSchool(lat, lng, level5)
}
}
def rets = []
if(minSchool){
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
}
if(rets.size()>0){
for(def d : rets){
if(!minSchool.school_address){
minSchool.school_address = d.address
}
if(!minSchool.attribute){
minSchool.attribute = d.attribute
}
}
}else{
if(minSchool){
if(minSchool.school_map_lng){
minSchool.school_lng2 = minSchool.school_map_lng
minSchool.school_lat2 = minSchool.school_map_lat
}else if(minSchool.school_lng1){
minSchool.school_lng2 = minSchool.school_lng1
minSchool.school_lat2 = minSchool.school_lat1
}else if(minSchool.school_lng){
minSchool.school_lng2 = minSchool.school_lng
minSchool.school_lat2 = minSchool.school_lat
}
}
}
return minSchool;
}
def findSchoolInfo(def city, def region, def schoolName){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("name", schoolName)
def dbcursor = school_col.find(q)
def rets = []
for(def obj : dbcursor){
rets << obj
}
return rets
}
def selectMinDistanceSchool(def lat, def lng, def schools){
def minSchool = null;
def minDistance = Integer.MAX_VALUE
for(def s1 : schools){
def lng2 = null
def lat2 = null
if(s1.school_map_lng){
lng2 = s1.school_map_lng
lat2 = s1.school_map_lat
}else if(s1.school_lng1){
lng2 = s1.school_lng1
lat2 = s1.school_lat1
}else if(s1.school_lng){
lng2 = s1.school_lng
lat2 = s1.school_lat
}
if(lat2){
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
println s1.district_name +" "+ s1.school_name+" "+distance
if(distance<minDistance){
minDistance = distance
minSchool = s1
minSchool.school_lng2 = lng2
minSchool.school_lat2 = lat2
}
}
}
return minSchool
}
def findPrimarySchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def findMiddleSchool(def city, def region, def name){
def q = new BasicDBObject()
q.put("match_city", city)
q.put("match_region", region)
q.put("match_name", name)
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
def datas = []
for(def db : dbcursor){
datas << db
}
return datas;
}
def getRegionCodeByCity(def city){
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
for(def d : datas){
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
}
}
}
@@ -11,7 +11,7 @@ class FieldStat {
}
def run(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row ->
def city = row['A']
if(city){
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
@@ -26,15 +26,14 @@ class FieldStat {
aliasTotalCount = aliasTotalCount + alias.size()
}
}
row['J'] = aliasTotalCount
row['K'] = aliasIndex
// row['J'] = aliasTotalCount
// row['K'] = aliasIndex
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
row['L'] = p+"%"
row['M'] = total
row['B'] = p+"%"
// row['M'] = total
println city +" "+p
}
});
}, 1);
println "finish"
}
}
@@ -0,0 +1,48 @@
package com.fangjia.script.gj.data
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class RegionPriceExport {
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
static main(args) {
def task = new RegionPriceExport();
task.run()
}
def run(){
def months = [202010,202011,202012,202101,202102,202103]
def regions = CommonUtils.getAllRegions("福州","region")
def rets = []
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
regions.each{region,values->
def data = []
data << "福州"
data << region
for(def month :months){
def dbo = findPrice("福州", region, month);
if(dbo){
data << dbo.avg_price
}else{
data << ""
}
}
rets << data
}
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
}
def findPrice(city, region, month){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("date", month)
def dbo = REGION_SELL_STAT.findOne(q)
return dbo
}
}
File diff suppressed because one or more lines are too long
@@ -0,0 +1,580 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.concurrent.atomic.AtomicLong
class CityStat {
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new CityStat()
// task.run()
// task.run1();
task.run3();
// task.run4()
// task.run6()
// task.run7()
// task.run8()
}
// def run8(){
// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d->
//
// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
// if(dist){
// def tradeAvgPrice = findTrades(dist, startDate, endDate);
// }
//
// },1, 0)
// }
def run5(){
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
def total = 0
def scount = 0
def index = 0
// final CountDownLatch latch = new CountDownLatch(districts.size())
for(def dist : dists){
index++
def rets = findTradeDetails(dist, "2020-03-24");
println index
if(rets){
total = total + rets.size()
def count = findSimialPriceCount(dist, rets);
scount = scount+count
}
}
println scount +" "+ total
def pre = DataHandleUtils.getDoubleRound(2, scount*100/total)
println pre;
}
def run7(){
def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"])
def distMap = [:]
for(def dist : dists){
if(dist.alias) {
if (!dist.alias.contains(dist.name)) {
dist.alias << dist.name
}
for(def name : dist.alias){
distMap.put( dist.region +"#"+name, dist)
}
}else{
distMap.put( dist.region +"#"+dist.name, dist)
}
}
def rets = findTradeDetails2("镇江","2020-03-24");
def total = 0
def scount = 0
def index = 0
for(def t : rets){
def key = t.region+"#"+t.district_name
def dist = distMap.get(key)
index++
println index
if(dist){
total++
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
if(t.avg_price && weekendPrice){
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
def m = Math.abs(mod)
if(m<=15){
scount++;
}
}
}
}
println scount +" "+total
}
def run6(){
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
final CountDownLatch latch = new CountDownLatch(dists.size())
def totalCount = new AtomicLong()
def tradeTotal = new AtomicLong()
def scount = new AtomicLong()
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def rets = findTradeDetails(dist, "2020-03-24");
if(rets){
// tradeTotal = tradeTotal + rets.size()
tradeTotal.addAndGet(rets.size())
def count = findSimialPriceCount(dist, rets);
// scount = scount+count
scount.addAndGet(count);
}
println totalCount.get() +" "+dist.name
}catch (Exception e){
}finally {
latch.countDown()
}
}
});
}
latch.await()
println scount +" "+ tradeTotal
def pre = DataHandleUtils.getDoubleRound(2, scount*100/tradeTotal)
println pre;
}
def findSimialPriceCount(def dist, def rets){
def count = 0
for(def t : rets){
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
if(t.avg_price && weekendPrice){
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
def m = Math.abs(mod)
if(m<=15){
count++;
}
}
// def weekend = DateUtils.getFirstDayOfWeek()
// def stat = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, weekend)
// if(t.avg_price && stat){
// def mod = (stat.avg_price - t.avg_price)*100/t.avg_price
// def m = Math.abs(mod)
// if(m<=15){
// count++;
// }
// }
}
return count;
}
def run3(){
ExcelUtils.multiThreadingModify("D:\\数据\\建行银行\\建行匹配小区17.xlsx",{data->
if(data['A']){
def startDate = DateUtils.addMonth(data['D'], -3)
def endDate = DateUtils.addMonth(data['D'], 3)
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
def tradeAvgPrice = findTrades(dist, startDate, endDate);
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
data['H'] = tradeAvgPrice.toInteger()
}
})
}
def run4(){
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
def startDate = DateUtils.addMonth(new Date(), -3)
def endDate = "2021-02-01"
def result = []
def allDistricts = []
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
allDistricts.addAll(districts)
println city +" "+districts.size()
}
def rets = findDistrictTrade(allDistricts, startDate,endDate);
println rets.size()
}
def findDistrictTrade(def districts, def startDate, def endDate){
def rets = []
final CountDownLatch latch = new CountDownLatch(districts.size())
def final AtomicInteger count_ = new AtomicInteger(0)
for(def i=0;i<districts.size;i++){
def dist = districts.get(i)
fixedThreadPool.execute(
new Runnable() {
void run() {
try{
println count_.incrementAndGet()+" "+dist.city+" "+dist.region+" "+dist.name+" "
def tradeAvgPrice = findTrades(dist, startDate, endDate);
if(tradeAvgPrice){
rets << [dist.city, dist.region, dist.name, tradeAvgPrice.toInteger()]
}
}catch (Exception e){
}finally {
latch.countDown()
}
}
}
)
}
latch.await()
ExcelUtils.write("D:\\数据\\深圳建行\\成交数据.xlsx",rets, "成交")
return rets;
}
def run(){
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
def rets = []
rets << ["城市","小区数量","均价覆盖率","样本计算小区均价覆盖率","badcase数量", "近三个月成交数据量","近三个月有成交数据的小区覆盖率"]
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
def priceCount = findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), null);
def houseSatCount = 0//findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), 2);
def badcaseCount = findBadCaseCount(city, DateUtils.parseFromTightByPattern("2020-11-29", "yyyy-MM-dd"));
def tradeCount = 0//findDealPriceCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"));
// def tradeDistCount = calTradeDistCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
def pricePercent = DataHandleUtils.getDoubleRound(2, priceCount*100/districts.size())
def houseStatPercent = 0//DataHandleUtils.getDoubleRound(2, houseSatCount*100/districts.size())
def badcasePercent = DataHandleUtils.getDoubleRound(2, badcaseCount*100/districts.size())
def tradeDistPercent = 0//DataHandleUtils.getDoubleRound(2, tradeDistCount*100/districts.size())
rets << [city, districts.size(), pricePercent+"%", houseStatPercent+"%", badcasePercent+"%", tradeCount, tradeDistPercent+"%"]
println city +" 完成"
}
ExcelUtils.write("d:\\数据\\统计\\城市统计1.xlsx", rets, "统计")
}
def run1(){
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
def citys = ["上海"]
def rets = []
rets << ["城市","区域","小区名","成交价","评估价"]
for(def city : citys) {
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
def retMap = calTradeDistDetail(city,DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
retMap.each {k,v->
if(v && v.size()>0){
def sum = v.sum()
def tradeAvgPrice = sum/v.size()
def arrs = k.split("#")
def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"))
rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price]
}
}
println city +" 完成"
}
ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交")
}
def run2(){
def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1)
def cityDataMap = [:]
def index = 0
for(def data : datas){
index++
println index+" "+data
def lists = cityDataMap.get(data['A'])
if(!lists){
lists = []
cityDataMap.put(data['A'], lists)
}
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
dist.calDate = data['D']
dist.evaluationPrice = data['E']
lists << dist
}
def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ]
def results = []
results << ["城市","区域","小区名","成交价","评估价"]
for(def city : citys){
def districts = cityDataMap.get(city)
def rets = findAvgPriceTrades(city, districts)
results.addAll(rets)
}
ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据")
}
def findAvgPriceTrades(city, districts){
def rets = []
final CountDownLatch latch = new CountDownLatch(districts.size())
for(def i=0;i<districts.size();i++){
def dist = districts[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
def startDate = DateUtils.addMonth(dist.calDate, -3)
def endDate = DateUtils.addMonth(dist.calDate, 3)
def tradeAvgPrice = findTrades(dist, startDate, endDate);
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
// def stat = findDistrictStat(city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2021-01-10", "yyyy-MM-dd"))
rets << [city, dist.region, dist.name, tradeAvgPrice.toInteger(), dist.evaluationPrice]
}
}
)
}
latch.await()
return rets;
}
def findTrades(dist, startDate, endDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def avg_price = 0
def total_price = 0
def count = 0
for(def d : dbcorsor){
if(d.avg_price){
total_price = total_price + d.avg_price
count++
}
}
if(count > 3){
avg_price = total_price/count
}
return avg_price
}
def findTradeDetails(dist, startDate){
def q = new BasicDBObject()
q.put("city", dist.city)
q.put("region", dist.region)
if(dist.alias){
if(!dist.alias.contains(dist.name)){
dist.alias << dist.name
}
q.put("district_name", new BasicDBObject('$in', dist.alias))
}else{
q.put("district_name", dist.name)
}
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcorsor){
if(d.avg_price){
rets << d
}
}
return rets
}
def findTradeDetails2(city, startDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def rets = []
for(def d : dbcorsor){
if(d.avg_price){
rets << d
}
}
return rets
}
def findStatCount(def city, def weekend, def value_type){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
if(value_type){
q.put("value_type", value_type)
}
def count = DIST_STAT_COLL.count(q)
return count;
}
def findDistrictStat(def city, def region, def name, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("region", region)
q.put("district_name", name)
q.put("weekend", weekend)
def obj = DIST_STAT_COLL.findOne(q)
return obj;
}
def findBadCaseCount(def city, def weekend){
def q = new BasicDBObject()
q.put("city", city)
q.put("weekend", weekend)
def count = BAD_CASE_COLL.count(q)
return count;
}
def findDealPriceCount(def city, def tradeDate){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def count = DEAL_PRICE_COLL.count(q)
return count;
}
def calTradeDistCount(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("source","链家在线")
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def rets = []
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist && !rets.contains(dist)){
rets << dist
}
}
return rets.size();
}
def findIsExistsDB(def region, def district_name, def distMap){
def dist = null;
def regionDists = distMap.get(region)
for(def rd : regionDists){
def alias = rd.alias
if(!alias){
alias = []
}
if(!alias.contains(rd.name)){
alias << rd.name
}
if(alias.contains(district_name)){
dist = rd
break;
}
}
return dist;
}
def calTradeDistDetail(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def retMap = [:]
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist){
def tradePrices = retMap.get(dist.region+"#"+dist.name)
if(!tradePrices){
tradePrices = []
retMap.put(dist.region+"#"+dist.name, tradePrices)
}
if(obj.avg_price){
tradePrices << obj.avg_price.toInteger()
}
}
}
return retMap;
}
def calTradeDistDetail2(def city, def tradeDate, def districts){
def q = new BasicDBObject()
q.put("city", city)
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
def dbcorsor = DEAL_PRICE_COLL.find(q)
def distMap = [:]
for(def dist :districts){
def lists = distMap.get(dist.region)
if(!lists){
lists = []
distMap.put(dist.region, lists);
}
lists << dist
}
def retMap = [:]
for(def obj : dbcorsor){
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
if(dist){
def tradePrices = retMap.get(dist.region+"#"+dist.name)
if(!tradePrices){
tradePrices = []
retMap.put(dist.region+"#"+dist.name, tradePrices)
}
if(obj.avg_price){
tradePrices << obj.avg_price.toInteger()
}
}
}
return retMap;
}
}
@@ -0,0 +1,111 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.gj.paimai.GaoDeUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import java.util.regex.Pattern
class CompanyAddressCompare {
static main(args) {
def task = new CompanyAddressCompare()
task.run()
}
def run(){
ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row->
if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){
def city = CommonUtils.parseCityInAddress(row["D"])
if(!city){
city = CommonUtils.parseCityInAddress(row["B"])
}
if(city){
def similarData = null
// if(isAddress(row["D"])){
def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"")
if(simpleAddress && simpleAddress.contains("号")){
simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1)
}
def pois = GaoDeUtils.search(simpleAddress, city, null)
similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null)
println row['C']+" "+pois.size()
// }
if(similarData){
row['F'] = similarData.address
row['G'] = similarData.name
row['H'] = similarData.score
}
}
}
}, 0)
}
def findSimilarDatas(pois, name, address, city, rMap){
def data = null
def maxScore = 0
for(def poi : pois){
if(!poi.cityname){
def m = getMapRegion(poi.location)
if(m){
poi.cityname = m.city
poi.pname = m.province
poi.township = m.township
poi.adname = m.district
}
}
if(poi.cityname.contains(city)){
def searchName = poi.name
def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name);
def addressScore = 0
if(poi.address){
def poiAddress = poi.address.replace("号楼","号")
addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address);
if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){
addressScore = addressScore + 0.5
}
}
def totalScore = nameScore + addressScore
if(totalScore > maxScore){
maxScore = totalScore
data = poi
data.score = totalScore
}
println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"")
}
}
return data;
}
def isAddress(address){
def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+";
def pattern = Pattern.compile(REG_ADDR1);
def matcher = pattern.matcher(address)
if(matcher.find()){
return true;
}
return false;
}
def getMapRegion(def location){
def jsonObj = GaoDeApiUtils.geocode(location)
def district = null
if(jsonObj && jsonObj.status=="1"){
def detailMap = jsonObj.regeocode.addressComponent
if(!detailMap.city){
if(detailMap.province && detailMap.province.toString().endsWith("市")){
detailMap.city = detailMap.province
}
}
return detailMap
}
return district;
}
}
@@ -0,0 +1,102 @@
package com.fangjia.script.gj.test
import com.fangjia.core.RegionData
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
class CrawlDistrictIdSelect {
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds")
static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds")
static main(args) {
def task = new CrawlDistrictIdSelect()
task.run()
// task.synch()
}
def synch(){
def q = new BasicDBObject()
def dbcursor = districtseeds2Coll.find(q)
def list = []
for(def d :dbcursor){
list << d
if(list.size()==10000){
districtseedsColl.insert(list)
list = []
}
}
if(list.size()>0){
districtseedsColl.insert(list)
}
}
def run(){
def datas = findCrawlData()
datas.each {k,v->
if(v){
for(def d : v){
d.c_date = new Date()
d.m_date = new Date()
districtseedsColl.save(d)
}
}
}
}
def findCrawlData(){
def q = new BasicDBObject()
q.put("city","福州")
q.put("source", "安居客")
q.put("s_date",new BasicDBObject('$gte', 202001))
def dbcursor = thirdPartyColl.find(q)
def retMap = [:]
def prefix = "/community/view/"
def keyList =[]
for(def db : dbcursor){
def url = null
if(!db.url){
if(db.district_id){
url = db.district_id
}
}else{
url = db.url
}
if(url && url.contains(prefix) ){
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
if(district_id=="0"){
continue
}
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
def region = RegionData.formatRegion(db.city, db.region)
def key = shortCity +"#"+district_id
def ids = retMap.get(key)
if(!ids){
ids = []
retMap.put(key, ids)
}
if(!keyList.contains(key)){
def map = new BasicDBObject()
map.root = "anjuke.com"
map.city_short= shortCity
map.city = db.city
map.region = region
map.district_id = district_id
map.source_url = url
ids << map
keyList << key
}
}else{
println url +" 链接不合要求"
}
}
return retMap
}
}
@@ -0,0 +1,101 @@
package com.fangjia.script.gj.test
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.QueryOperators
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
class ExportToDealPrice {
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
final AtomicInteger count = new AtomicInteger()
static main(args) {
def task = new ExportToDealPrice();
task.run()
}
def run(){
def datas = findDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def data = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
count.addAndGet(1)
def dbo = addPrice(data)
if(dbo){
data.weekend_avg_price = dbo.avg_price
}
}catch(Exception e){
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
save(datas);
}
def save(def datas){
def ret = [];
def index = 0;
for(def dbo : datas){
ret << dbo;
index++;
if(ret.size()%10000==0){
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
TRADE_MATCH_PRICE_COLL.insert(ret);
ret = [];
}
}
if(ret.size()>0){
TRADE_MATCH_PRICE_COLL.insert(ret);
}
}
def addPrice(data){
def weekend = DateUtils.getFirstDayOfWeek(data.trade_date)
def q = new BasicDBObject()
q.put("city", data.match_city)
q.put("region", data.match_region)
q.put("district_name", data.match_name)
q.put("weekend", weekend)
def dbo = DISTRICT_STAT_COLL.findOne(q)
if(!dbo){
q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend))
dbo = DISTRICT_STAT_COLL.findOne(q)
}
println count.get()+" "+data.city +" "+data.match_name
return dbo
}
def findDatas(){
def q = new BasicDBObject()
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd")))
def dbcursor = DEAL_PRICE_MATCH_COLL.find(q)
def rets = []
for(def db : dbcursor){
rets << db
}
return rets;
}
}
@@ -0,0 +1,182 @@
package com.fangjia.script.gj.test
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.fangjia.core.util.BaiDuApiUtils
import com.mongodb.BasicDBObject
import com.fangjia.script.util.DbUtils
class FZloationUpdate {
static main(args) {
def task = new FZloationUpdate();
task.run()
}
def run(){
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
def index = 0
def locationMap = getMoreSameLocationDist(districts)
def total = 0
locationMap.each {k,v->
total = updateLocation(v, total);
println "---------------------------------------------------------------------------------------------------------------"
}
}
def updateLocation(def lists, def total){
for(def dist : lists){
def addrDistMap = getLocation(dist, dist.address)
def nameDistMap = getLocation(dist, dist.region + dist.name)
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
if(addrDistMap){
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
def baidu = null
def lat = null
def lng = null
def upldateFlag = ""
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
lat = addrDistMap.lat
lng = addrDistMap.lng
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
upldateFlag = "Address"
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
lng = nameDistMap.lng
lat = nameDistMap.lat
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
upldateFlag = "Name"
}
if(baidu){
total++
def udata = new BasicDBObject()
udata.put("m_date", new Date())
Long baiduLat = baidu.y*10000000000L
Long baiduLng = baidu.x*10000000000L
udata.put("lat2", baiduLat)
udata.put("lng2", baiduLng)
Long lat0 = lat*10000000000L
Long lng0 = lng*10000000000L
udata.put("lat", lat0)
udata.put("lng", lng0 )
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
}
}
}
}
return total
}
def getMoreSameLocationDist(def districts){
def locationMap = [:]
for(def i=0;i<districts.size();i++){
def dist = districts[i]
def lat = dist.lat/10000000000.0;
def lng = dist.lng/10000000000.0;
def locationKey = lng + "-" + lat
def list = locationMap.get(locationKey)
if(!list){
list = []
locationMap.put(locationKey, list);
}
list << dist
}
def resMap = [:]
locationMap.each{k,v->
if(v.size()>=2){
v.sort{ a,b->
return a.name.size().compareTo(b.name.size())
}
resMap.put(k,v)
}
}
def lastRetMaps = [:]
resMap.each{k,v->
def swname = null
def count = 0
for(def i=0;i<v.size();i++){
def d = v[i]
if(i==0){
if(d.name.contains("期")||d.name.contains("区")){
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
}else{
swname = d.name
}
}else{
if(d.name.contains(swname)||swname.contains(d.name)){
count++
}
}
}
if(count+1!=v.size()){
lastRetMaps.put(k,v)
}
}
return lastRetMaps;
}
def deleteRegion(def s, def region){
def key = ""
if(s.indexOf(region)>-1){
if(s.indexOf(region+"区")>-1){
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
}else if(s.indexOf(region+"县")>-1){
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
}else if(s.indexOf(region+"市")>-1){
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
}
}else{
key = s;
}
return key
}
def getLocation(dist, addr){
if(addr){
def rMap = [:]
def reps = GaoDeApiUtils.geo(dist.city, addr);
if(reps && reps.count.toInteger()>=1){
def map = reps.geocodes.get(0)
def location = map.location
def level = map.level
if(location){
def locMap = location.split(",")
def lat = dist.lat/10000000000.0;
def lng = dist.lng/10000000000.0;
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
rMap.distance = distance
rMap.level = level
rMap.lat = locMap[1].toDouble()
rMap.lng = locMap[0].toDouble()
rMap.keyword = map.formatted_address
rMap.location = location
rMap.region = map.district
return rMap;
}
}
}
return null;
}
}
@@ -0,0 +1,4 @@
package com.fangjia.script.gj.test
class FzLocationBatch {
}
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
// task.matchHouse();
// task.villa()
task.addressParse()
// task.addressParse()
task.deleteProcess()
}
def repeat(){
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
return floor;
}
def deleteProcess(){
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
def addrList = []
for(def d : datas){
def addr = d['G']
addrList << addr
}
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
def addr = row['C']
if(addrList.contains(addr)){
row['R'] = "已调查"
}
},0);
}
}
@@ -0,0 +1,7 @@
package com.fangjia.script.gj.test
import java.text.SimpleDateFormat
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
Date date = new Date(1617211269868);
print dateFormat.format(date);
@@ -18,8 +18,7 @@ import org.apache.commons.lang.StringUtils
class AutocompleteDataUpdaterBatch extends BaseBatch {
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "192.168.0.51", 29009)
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "mongo.db.fangjia.com")
private static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
private static final DSCollection AUTOCOMPLETE_COLL = new DataStore("192.168.0.38", 27008).getCollection("fangjia.autocomplete")
private static final long C_DATE = DateUtil.formatToYYYYMMDDhhmmss(new Date()).toLong()
@@ -0,0 +1,156 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import org.apache.commons.lang.StringUtils
/**
* 抓取总量监控
*
* @author shenjian
* @since 2021/5/12
*/
@BatchOwner(Watcher.ShenJian)
class SeawaterMonitor extends BaseBatch {
static final Long expectNumber = 400000;
// 运维组 运维警报
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
// 重点城市
static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州",
"沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州",
"南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州",
"厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳",
"武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德",
"海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海",
"西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理",
"东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴",
"淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远",
"马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照",
"拉萨","大同","文昌","陵水","保亭"];
// 重点网站
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
static void main(String[] args) {
runBatch(new SeawaterMonitor(), args)
}
/**
* 监控前一日抓取量
* @return
*/
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
def total
String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' "
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
if (total < expectNumber) {
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
}
return ""
}
/**
* 统计重点城市近半月抓取量
* @return
*/
static boolean countCityCrawlNumber(def startDate, def endDate) {
def result = []
result << ['城市', '抓取量']
// 统计近半个月各重点城市抓取量
def cityFormatted = city.join("','")
String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
ElasticUtils.executeSqlQuery(sql,{data->
result << [data.city, data.total]
})
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
}
/**
* 统计前一日重点城市抓取量,昨天为零则警报
* @return
*/
static def monitorCityCrawlNumber(def startDate, def endDate) {
def msg = "【重点城市抓取异常】"
def set = new HashSet()
def cityFormatted = city.join("','")
String sql = "select * from " +
" (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 "
//" order by total "
ElasticUtils.executeSqlQuery(sql,{data->
set.add(data.city)
})
def unNormal = []
for (def info : city) {
if (!set.contains(info)) {
unNormal.add(info)
}
}
msg += unNormal.join("、") + " 抓取量为0"
if (unNormal.size() > 0) {
return msg
}
return ""
}
/**
* 统计重点网站抓取情况,近一周为0则警报
*/
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
def msg = "【重点网站抓取异常】"
def set = new HashSet()
def siteFormatted = site.join("','")
String sql = "select * from " +
" (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
//" order by total "
ElasticUtils.executeSqlQuery(sql,{data->
set.add(data.root)
})
def unNormal = []
for (def info : site) {
if (!set.contains(info)) {
unNormal.add(info)
}
}
msg += unNormal.join("、") + " 近一周抓取量为0"
if (unNormal.size() > 0) {
return msg
}
return ""
}
@Override
void process(Object args) {
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1))
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime();
def endDate = DateUtil.truncateDay(new Date()).getTime()
def result = []
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
msg = monitorCityCrawlNumber(startDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
if (result.size() > 0) {
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
println(msg)
DingDingMessageUtil.sendTextMessage(access_token, msg)
}
}
}
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
def run(){
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
for(def source :sources){
def count1 = findTotalCount(source, 202101);
def count2 = findTotalCount(source, 202102);
def count1 = findTotalCount(source, 202102);
def count2 = findTotalCount(source, 202103);
println source +" "+count1 +" "+count2
}
}
@@ -0,0 +1,213 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import net.sf.json.groovy.JsonSlurper
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
class AddFangSchoolRegion {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
def task = new AddFangSchoolRegion();
//task.process()
//task.match()
task.addMapLocation()
}
def addMapLocation(){
def q = new BasicDBObject()
def dbcursor = district_school_col.find(q)
def index = 0
for(def d : dbcursor){
def data = findSchool(d.city, d.region, d.school_name)
if(data && data.location){
def location = data.location
if(location){
def udata = new BasicDBObject()
udata.put("school_map_lng", location[0])
udata.put("school_map_lat", location[1])
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
}
}
index ++
println index
}
}
def match(){
def datas = findAllDatas()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size;i++){
def data = datas[i]
fixedThreadPool.execute(
new Runnable() {
void run() {
try {
// if(!data.match_name){
println i +" "+data.district_name
matching(data);
// }
} catch (Exception e) {
println "Òì³£"
e.printStackTrace();
} finally {
latch.countDown()
}
}
}
);
}
latch.await()
}
def matching(def it){
def rMap = matchDistrict(it.city, it.region, it.district_name)
if(!rMap){
rMap = matchDistrict(it.city, it.region, it.address)
}
if(rMap){
DBObject data = new BasicDBObject()
data.put("match_city", rMap.city)
data.put("match_region", rMap.region)
data.put("match_name", rMap.name)
data.put("match_address", rMap.address)
data.put("match_lat", rMap.lat)
data.put("match_lng", rMap.lng)
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
}
}
def matchDistrict(def city, def region, def address){
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
String content = new HttpRequest().get(url, "utf-8");
def rMap = [:]
if(content){
def doc = new JsonSlurper().parseText(content)
if(doc && doc.code ==200 && doc.result){
def dto = doc.result
if(dto && dto.searchAddress){
def ret = dto.matchResult
if(ret.match){
rMap.city = dto.city
rMap.region = dto.district
rMap.name = dto.name
rMap.lat = dto.lat
rMap.lng = dto.lng
rMap.id = dto.id
rMap.address = dto.address
}
}
}
}
return rMap;
}
def process(){
def datas = findNoRegionData();
findSimialData(datas);
}
def findAllDatas(){
def q = new BasicDBObject()
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
def dbcursor = district_school_col.find(q)
def rets =[]
for(def d : dbcursor){
rets << d
}
return rets;
}
def findNoRegionData(){
def q = new BasicDBObject()
q.put("region", null)
def dbcursor = district_school_col.find(q)
def rets = []
for(def d : dbcursor){
rets << d
}
return rets;
}
def findSimialData(datas){
for(def d : datas){
def obj = findThirdPriceData(d.city, d.url+"/")
if(!obj){
obj = findThirdPriceByName(d.city, d.district_name)
}
if(obj){
d.region = obj.region
if(!d.district_addrss){
d.district_address = obj.address
}
updateDistrictSchool(d);
}else{
println d.school_name+"" + d.district_name+"û²éµ½"
}
}
}
def findThirdPriceData(def city, def url){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("district_id", url)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def findThirdPriceByName(def city, def name){
def q = new BasicDBObject()
q.put("source","ËÑ·¿")
q.put("city", city)
q.put("name", name)
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
def obj = null;
for(def d : dbcursor){
obj = d
}
return obj;
}
def updateDistrictSchool(d){
DBObject data = new BasicDBObject()
data.put("region", d.region)
data.put("district_address", d.district_address)
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
}
def findSchool(city, region, school_name){
DBObject data = new BasicDBObject()
data.put("city", city)
data.put("region", region)
data.put("name", school_name)
return school_col.findOne(data)
}
}
@@ -0,0 +1,52 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.fangjia.script.util.ABuYunUtil
import org.apache.commons.io.IOUtils
import org.apache.http.HttpResponse
import org.apache.http.client.HttpClient
import org.apache.http.client.methods.HttpGet
import org.apache.http.impl.client.HttpClients
class AnjukeCrawl {
static main(args) {
def task = new AnjukeCrawl();
task.run()
}
def run(){
HttpClient client = HttpClients.createDefault();
HttpGet get = new HttpGet("https://m.anjuke.com/cityList/");
// get.setHeader("Upgrade-Insecure-Requests", "1");
// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
// get.setHeader("Host","m.anjuke.com");
// get.setHeader("Connection","keep-alive");
// get.setHeader("Cache-Control","max-age=0");
// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"');
// get.setHeader("sec-ch-ua-mobile","'?0");
// get.setHeader("Upgrade-Insecure-Requests","1");
def header = [:]
header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36");
header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
// get.setHeader("Sec-Fetch-Site","none");
// get.setHeader("Sec-Fetch-Mode","navigate");
// get.setHeader("Sec-Fetch-User","?1");
// get.setHeader("Sec-Fetch-Dest","document");
//// get.setHeader("Accept-Encoding"," gzip, deflate, br");
// get.setHeader("Accept-Language","zh-CN,zh;q=0.9");
def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8")
printl s1
HttpResponse response = client.execute(get);
def s = getResponseBodyAsString(response)
println s
}
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
}
}
@@ -0,0 +1,344 @@
package com.fangjia.script.thirdPriceCrawl.other
import com.alibaba.fastjson.annotation.JSONField
import com.fangjia.core.RegionData
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.util.ABuYunUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.httpclient.NameValuePair
import org.apache.commons.lang.StringUtils
import org.apache.http.HttpEntity
import org.apache.http.client.entity.UrlEncodedFormEntity
import org.apache.http.client.methods.CloseableHttpResponse
import org.apache.http.client.methods.HttpPost
import org.apache.http.impl.client.CloseableHttpClient
import org.apache.http.impl.client.HttpClients
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import java.nio.charset.Charset
import java.util.zip.GZIPInputStream
class GushuwangCrawl {
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
static main(args) {
def task = new GushuwangCrawl();
task.process()
}
void process(Object args) {
// chaxuequCrawl()
suzhouCrawl();
}
def suzhouCrawl(){
def page = 1
def stop = false
def list = []
while(!stop){
def s = schoolList(page);
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.ok){
def datas = jsonObject.extra.list
if(datas){
list.addAll(datas)
page++
println page
}else{
stop = true
}
}else{
stop = true
}
}
}
if(list){
def schools = []
for(def d : list){
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = "苏州"
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(d.schoolname)
schoolData.name = d.schoolname
schoolData.address = d.adr
/**来源网站*/
schoolData.source = "苏州学区查询"
/**房源详情url*/
schools << schoolData
}
saveChaxuequSchool(schools)
}
}
def chaxuequCrawl(){
def page = 1
def stop = false
def schools = []
while(!stop){
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
println "内容:"+s
if(s){
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject.code == 200){
def data = jsonObject.data
def totalPage = data.pages
if(data.list){
page++
if(page+1>totalPage){
stop = true;
}
schools.addAll(data.list)
}
}
}
}
print(schools.size());
def districtSchools = []
def schoolInfos = []
if(schools.size()>0){
def url = "https://51yongka.com/fapi/school/details"
for(def d : schools){
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
def s = chaxuequInfoCrawl(url, params)
JSONObject jsonObject = JSONObject.fromObject(s);
if(jsonObject && jsonObject.code==200){
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
def districts = jsonObject.data.nearbyPropertyList
for(def dist : districts){
def dataMap = new BasicDBObject()
dataMap.city = jsonObject.data.city
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
dataMap.category = converCategory(jsonObject.data.schoolName)
dataMap.district_name = dist.name.trim()
dataMap.district_address = dist.address
dataMap.school_name = jsonObject.data.schoolName.trim()
dataMap.school_address = jsonObject.data.address
dataMap.source = "查学区房"
dataMap.url = url + "?"+params
dataMap.telephone = jsonObject.data.officialTelephone;
districtSchools << dataMap
}
}else{
def schoolData = new BasicDBObject()
schoolData.province = "江苏"
schoolData.city = jsonObject.data.city
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
/**类型: 幼儿园,小学,初中*/
schoolData.category = converCategory(jsonObject.data.schoolName)
schoolData.name = jsonObject.data.schoolName.trim()
schoolData.address = jsonObject.data.address
/**来源网站*/
schoolData.source = "查学区房"
/**房源详情url*/
schoolData.telephone = jsonObject.data.officialTelephone;
schoolInfos << schoolData
println jsonObject.data.schoolName +" 无周边楼盘"
}
}
println s;
}
}
println "小区:"+districtSchools.size()
// saveChaxuequDistrictSchool(districtSchools)
saveChaxuequSchool(schoolInfos)
}
def saveChaxuequSchool(schoolInfos){
for(def ds : schoolInfos){
def obj = findSchoolData(ds.city, ds.school_name);
if(!obj){
school_col.save(ds)
}
}
}
def saveChaxuequDistrictSchool(def districtSchools){
for(def ds : districtSchools){
def obj = findDistrictSchoolData(ds.city, ds.school_name);
if(!obj){
district_school_col.save(ds)
}
}
}
def findSchoolData(city, school_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
return school_col.findOne(q)
}
def findDistrictSchoolData(city, school_name, district_name){
def q = new BasicDBObject()
q.put("city", city)
q.put("school_name", school_name)
q.put("district_name", district_name)
return district_school_col.findOne(q)
}
def converCategory(def schoolName){
if(schoolName){
if(schoolName.contains("初中部")||schoolName.contains("中学")){
return "初中"
}else if(schoolName.contains("小学")){
return "小学"
}
}
return null
}
def static chaxuequInfoCrawl(url, bodyquery){
def responseMap = [:]
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","51yongka.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","75")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
headers.put("Accept-Encoding","gzip, deflate, br")
String s = doPost(url, bodyquery, headers)
return s
} catch (Exception e) {
e.printStackTrace()
}
return null
}
def static schoolList(page) {
def responseMap = [:]
String s = null
try {
//headers信息
Map<String, String> headers = new LinkedHashMap<String, String>()
headers.put("Host","m.gusuwang.com")
headers.put("Connection","keep-alive")
headers.put("Content-Length","40")
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
headers.put("content-type","application/x-www-form-urlencoded")
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
// headers.put("Accept-Encoding","gzip, deflate, br")
//params信息
Map<String, String> params = new HashMap<String, String>();
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
// //post请求
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
} catch (Exception e) {
e.printStackTrace()
}
return s
}
static String doPost(String strUrl, String content, Map headers) {
String result = "";
OutputStream outputStream = null;
InputStream inputStream = null;
StringBuilder sb = new StringBuilder();
try {
URL url = new URL(strUrl);
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
urlConnection.setRequestMethod("POST");
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setConnectTimeout(30000);
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
urlConnection.setReadTimeout(30000);
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
urlConnection.setDoOutput(true);
urlConnection.setDoInput(true);
//设置通用请求属性为默认浏览器编码类型
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
if (headers != null && headers.keySet().size() > 0) {
for (String head : headers.keySet()) {
urlConnection.setRequestProperty(head, headers.get(head));
}
}
//getOutputStream()返回的输出流,用于写入参数数据。
outputStream = urlConnection.getOutputStream();
outputStream.write(content.getBytes());
outputStream.flush();
outputStream.close();
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
inputStream = urlConnection.getInputStream();
// byte[] data = new byte[1024];
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
// while (inputStream.read(data) != -1) {
// String s = new String(data, Charset.forName("utf-8"));
// sb.append(s);
// }
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
char[] buff = new char[1024];
int length = 0;
while ((length = reader.read(buff)) != -1) {
String s = new String(buff, 0, length);
sb.append(s);
}
} catch (IOException e) {
e.printStackTrace();
}finally{
if(outputStream!=null){
outputStream.close();
}
if(inputStream!=null){
inputStream.close();
}
}
return sb.toString();
}
}
@@ -425,6 +425,7 @@ class ABuYunUtil {
buffer.append(bf, 0, len);
}
} catch (IOException e) {
e.printStackTrace()
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
} finally {
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())
@@ -44,7 +44,7 @@ class ElasticUtils {
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
Properties connectionProperties = new Properties();
connectionProperties.setProperty("user","developer");
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
@@ -52,7 +52,7 @@ class ElasticUtils {
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
@@ -140,13 +140,12 @@ class ElasticUtils {
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
@@ -1,19 +1,17 @@
package com.fangjia.script.util
import com.mongodb.DBCursor
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
class HouseUtils{
static main(args){
@@ -121,9 +121,9 @@ public class OpenApiClient {
params.put("suggestName", suggestName)
}
String content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8")
while (content == null) {
content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8")
println("${city} ${input} ${cat}程序调用异常")
Thread.sleep(6000)
}
File diff suppressed because it is too large Load Diff
@@ -6,34 +6,50 @@ import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
def count = 0
List<List<String>> data = Collections.synchronizedList([])
data << ["ID","城市","区域","小区名","经度","纬度","街道"]
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Êý¾Ý.xlsx",{List item->
// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){
// item["D"] = "´æÔÚ"
// }else{
// item["D"] = "²»´æÔÚ"
// }
//})
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com")
ProvinceData.getAllCity("¸£½¨").each {def city->
int count = 0
def s = new TreeMap()
DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat->
count ++
int credit = (stat.credit * 100)/10
s[credit] = s.get(credit, 0) + 1
}
def data = []
data << city
data << count
for(int i=0; i <=10; i++){
data << s.get(i, 0)
}
println(data.join(","))
CommonUtils.findAllCity().each {def city->
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
if(StringUtils.isBlank(item.township) && item.lng){
count++
def lng = item.lng/10000000000d
def lat = item.lat/10000000000d
def location = "${lng},${lat}"
JSONObject resp = GaoDeApiUtils.geocode(location)
def row = []
row << item._id.toString()
row << item.city
row << item.region
row << item.name
row << lng
row << lat
BasicDBObject dbo = new BasicDBObject()
dbo.id = item._id.toString()
dbo.city = item.city
dbo.region = item.region
dbo.name = item.name
dbo.lng = lng
dbo.lat = lat
if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
row << resp.regeocode.addressComponent.township
dbo.township = resp.regeocode.addressComponent.township
}
println(row)
data << row
SEAWEED_TOWNSHIP.insert(dbo)
}
})
}
ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
@@ -1,6 +1,8 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
@@ -9,294 +11,33 @@ import net.sf.json.JSONObject
import java.util.regex.Matcher
import java.util.regex.Pattern
//
//def map = [:]
//ExcelUtils.read("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批0829.xlsx", 1).each {List item->
// if(item["AF"] && item["AH"]){
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
// map.get(key, []) << key
// }
//}
//ExcelUtils.modify("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批(房价网交付0905).xlsx", { List item ->
//// if(item["AF"] && item["AH"]){
//// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
//// List nums = map.get(key)
//// item["AT"] = nums.size()
//// }
//
// def city = "上海"
// def addr = item["AC"]
// if(addr){
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
// println searchAddress
//// item["AD"] = searchAddress.province
//// item["AE"] = searchAddress.city
//// item["AF"] = searchAddress.district
// if(searchAddress.standardAddress){
//// item["AG"] = searchAddress.street
//// if(searchAddress.streetNumber){
//// item["AG"] = item["AG"] + searchAddress.streetNumber
//// }
//
// if(searchAddress.township){
// item["AP"] = searchAddress.township
// }
// }
//
// if(item["AG"]){
// def addrMap = getRoad(item["AG"])
// if(addrMap){
// if(addrMap.road){
// item["AQ"] = addrMap.road
// }
// if(addrMap.roadNumber){
// item["AR"] = addrMap.roadNumber
// }
// }
// }
// }
//
//
//// if (item["AH"] && item["AF"]) {
//// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["AE"], item["AF"], item["AH"]))
//// if (seaweed) {
//// if (item["AR"] && item["AS"]) {
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2, item["AS"].toInteger(), item["AR"].toInteger())
//// if (eva) {
//// item["AL"] = eva.avgPrice
//// item["AM"] = eva.totalPrice
//// }
//// } else {
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2)
//// if (eva) {
//// item["AL"] = eva.avgPrice
//// item["AM"] = eva.totalPrice
//// }
//// }
//// }
////
//// }
//
//},1)
// if(item["AK"] =~ /^\d{4}$/){
// item["AR"] = item["AK"].toString().substring(0, 2)
// }else if(item["AK"] =~ /^\d{3}$/){
// item["AR"] = item["AK"].toString().substring(0, 1)
// }
// if(item["AR"] && item["AH"]){
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AS"] = nums[nums.size() - 1]
// item["AT"] = nums.unique().size()
// }
// println(item)
// item["AG"] = null
// def city = "上海"
// def addr = item["AC"]
// def area = item["o"]
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
// item["AD"] = searchAddress.province
// item["AE"] = searchAddress.city
// item["AF"] = searchAddress.district
// if(searchAddress.standardAddress){
// item["AG"] = searchAddress.street
// if(searchAddress.streetNumber){
// item["AG"] = item["AG"] + searchAddress.streetNumber
// }
// }
//
//
// if(resp.getDouble("credit") > 0){
// item["AE"] = resp.city
// item["AF"] = resp.district
// item["AH"] = resp.name
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
// item["AL"] = eva.avgPrice
// item["AM"] = eva.totalPrice
// item["AN"] = new Date().format("yyyy-MM-dd")
// if(!item["AG"]){
// item["AG"] = resp.address
// }
// }
// item["AI"] = searchAddress.buildingNumber
// item["AJ"] = searchAddress.unitNumber
// item["AK"] = searchAddress.roomNumber
// println(item)
//},1)
//
def sheetIndexs = ["湘潭": 0, "嘉兴": 1, "苏州": 2]
sheetIndexs.each { String city, int sheetIndex ->
// def map = [:]
// ExcelUtils.read("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批0828(测试)1.xlsx", sheetIndex).each {List item->
// if(item["Q"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// map.get(key, []) << key
// }
// }
ExcelUtils.multiThreadingModify("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批(房价网交付0905).xlsx", { List item ->
// if(item["Q"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AE"] = nums.size()
// }
// item["W"] = null
// item["X"] = null
// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["P"], item["Q"], item["S"]))
// if(seaweed){
// if(item["AC"] && item["AD"]){
// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2, item["AD"].toInteger(), item["AC"].toInteger())
// if(eva){
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// }
// }else{
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2)
// if(eva){
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// }
// }
// }
// }
// if(item["AC"] && item["S"]){
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
// List nums = map.get(key)
// nums.sort()
// item["AD"] = nums[nums.size() - 1]
// item["AE"] = nums.unique().size()
// }
// if(item["V"] =~ /^\d{4}$/){
// item["AC"] = item["V"].toString().substring(0, 2)
// }else if(item["V"] =~ /^\d{3}$/){
// item["AC"] = item["V"].toString().substring(0, 1)
// }else if(item["V"] =~ /^\d{6,7}$/){
// item["AC"] = item["V"].toString().substring(0, 2).replaceAll("^0", "")
// }
def addr = item["B"].toString().replaceAll("(", "").replace('区/县', "区").replace("街道/镇", "镇")
.replaceAll(")", "").replaceAll("【.*】", "").replaceAll("室/单元", "号")
// def area = item["L"]
def region = null
if (city == "嘉兴") {
region = "嘉善"
} else if (city == "苏州") {
region = "昆山"
}
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property", false, region)
JSONObject searchAddress = resp.getJSONObject("searchAddress")
def township = ""
if(searchAddress && searchAddress.township){
item["AP"] = searchAddress.township
township = searchAddress.township
}
if(item["S"]){
def addrMap = getRoad(item["S"])
if(addrMap){
if(addrMap.road){
def s = addrMap.road;
if(township){
if(township.contains(s)){
item["AQ"] = s.replace(township, "")
}else{
if(township.contains("镇")){
township = township.replace("镇","")
}
item["AQ"] = s.replace(township, "")
}
}else{
item["AQ"] = addrMap.road
}
}
if(addrMap.roadNumber){
item["AR"] = addrMap.roadNumber
}
}
}
println searchAddress
// item["O"] = searchAddress.province
// item["P"] = searchAddress.city
// item["Q"] = searchAddress.district
// if (searchAddress.standardAddress) {
// item["R"] = searchAddress.street
// if (searchAddress.streetNumber) {
// item["R"] = item["R"] + searchAddress.streetNumber
// }
// }
// if (resp.getDouble("credit") > 0) {
// item["O"] = resp.province
// item["P"] = resp.city
// item["Q"] = resp.district
// item["S"] = resp.name
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
// item["W"] = eva.avgPrice
// item["X"] = eva.totalPrice
// item["Y"] = new Date().format("yyyy-MM-dd")
// if (!item["R"]) {
// item["R"] = resp.address
// }
// }
// item["T"] = searchAddress.buildingNumber
// item["U"] = searchAddress.unitNumber
// item["V"] = searchAddress.roomNumber
// if(city == "湘潭"){
// String roomNumber = DataUtil.match(addr, Pattern.compile("\\d+(号|(号房))\$"))
// if(roomNumber){
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
// }
// roomNumber = DataUtil.match(addr, Pattern.compile("\\d{6,8}\$"))
// if(roomNumber){
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
// }
//
// }
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
new File("D:\\Êý¾Ý\\ѧУÊý¾Ý").eachFile {File file->
ExcelUtils.read(file.getAbsolutePath(), 0).each {List item->
BasicDBObject dbo = new BasicDBObject()
dbo.city = item["A"]
dbo.region = item["B"]
dbo.name = item["C"]
dbo.address = item["E"]
dbo.attribute = item["G"]
dbo.property = item["H"]
dbo.category = "Сѧ"
dbo.location = [DataUtil.roundDouble(item["I"], 6), DataUtil.roundDouble(item["J"], 6)]
school.insert(dbo)
println(dbo)
}
ExcelUtils.read(file.getAbsolutePath(), 1).each {List item->
BasicDBObject dbo = new BasicDBObject()
dbo.city = item["A"]
dbo.region = item["B"]
dbo.name = item["C"]
dbo.address = item["E"]
dbo.attribute = item["G"]
dbo.property = item["H"]
dbo.category = "ÖÐѧ"
dbo.location = [item["I"], item["J"]]
school.save(dbo)
println(item)
}, sheetIndex)
}
def getRoad(def s){
def rex = ".*(路|街道|街|大道|巷|道|里).*\\d+(弄|号)\$"
Pattern pattern = Pattern.compile(rex);
Matcher matcher = pattern.matcher(s);
List<String> list = new ArrayList();
while (matcher.find()) {
list.add(matcher.group());
}
if(list && list.size()>0){
def s1 = list[0]
def road = getRexText(s1, ".*(路|街道|街|大道|巷|道|里)")
def num = getRexText(s1, "\\d+")
return ["road":road, "roadNumber":num]
}else{
def road = getRexText(s, ".*(路|街道|街|大道|巷|道|里)")
return ["road":road]
}
return null
}
def getRexText(s, regx){
Pattern pattern = Pattern.compile(regx);
Matcher matcher = pattern.matcher(s);
while (matcher.find()) {
return matcher.group();
}
return null;
}
@@ -1,9 +1,16 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang3.StringUtils
import org.openqa.selenium.json.Json
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
@@ -30,8 +37,6 @@ import org.apache.commons.lang3.StringUtils
//}, 0,"结果")
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
// if (item["T"] == null || ![1, 2, 4].contains(item["T"].toString().toDouble().toInteger())) {
// return
@@ -114,43 +119,80 @@ import org.apache.commons.lang3.StringUtils
// println("${key} ${names.join(",")}")
// }
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\µØÖ·±ê×¼»¯.xlsx",{List item->
String address = "";
if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
address = ProvinceData.getDisplay(item["B"])
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\地址标准化.xlsx",{List item->
// String address = "";
// if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
// address = ProvinceData.getDisplay(item["B"])
// }
// if(StringUtils.isNotBlank(item["C"])){
// String cityDisplay = CityData.getCityDisplay(item["C"]);
// if(cityDisplay != null){
// address = address + cityDisplay
// }else{
// address = address + item["C"] + "市"
// }
// }
// if(StringUtils.isNotBlank(item["D"])){
// String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
// if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
// address = address + orgDistrict
// }
// }
// if(StringUtils.isNotBlank(item["E"])){
// address = address + item["E"]
// }
// if(StringUtils.isNotBlank(item["F"])){
// address = address + item["F"]
// }
// if(StringUtils.isNotBlank(item["G"])){
// address = address + item["G"]
// }
// if(StringUtils.isNotBlank(item["H"])){
// address = address + item["H"] + "栋"
// }
// if(StringUtils.isNotBlank(item["i"])){
// address = address + item["i"] + "单元"
// }
// if(StringUtils.isNotBlank(item["J"])){
// address = address + item["J"] + "室"
// }
// item["K"] = address
// println(address)
//})
def school = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia", "guiding_price")
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\深圳市住建局二手房指导价.xlsx", { List item ->
if (item["G"]) {
BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["G"], item["H"], item["I"]))
BasicDBObject guid = new BasicDBObject()
guid.guiding_type = 0
guid.estate_type = seaweed.estate_type2
guid.house_type = 0
guid.c_date = new Date()
guid.guiding_person = "深圳房管局"
guid.guiding_date = Date.parse('yyyyMMdd', '20210411')
guid.category = "district"
guid.guiding_value = (int)item["E"]
guid.name = item["D"]
guid.city = item["G"]
guid.region = item["H"]
guid.name = item["i"]
school.insert(guid)
println(guid)
}
if(StringUtils.isNotBlank(item["C"])){
String cityDisplay = CityData.getCityDisplay(item["C"]);
if(cityDisplay != null){
address = address + cityDisplay
}else{
address = address + item["C"] + "ÊÐ"
}
}
if(StringUtils.isNotBlank(item["D"])){
String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
address = address + orgDistrict
}
}
if(StringUtils.isNotBlank(item["E"])){
address = address + item["E"]
}
if(StringUtils.isNotBlank(item["F"])){
address = address + item["F"]
}
if(StringUtils.isNotBlank(item["G"])){
address = address + item["G"]
}
if(StringUtils.isNotBlank(item["H"])){
address = address + item["H"] + "¶°"
}
if(StringUtils.isNotBlank(item["i"])){
address = address + item["i"] + "µ¥Ôª"
}
if(StringUtils.isNotBlank(item["J"])){
address = address + item["J"] + "ÊÒ"
}
item["K"] = address
println(address)
// if(!item["F"]){
// JSONObject resp = OpenApiClient.addressMatch("深圳", item["D"], 'property', false, item["B"])
// if(resp.id){
// BasicDBObject seaweed = DbUtils.seaweed.findById(resp.id)
// if(seaweed){
// item["F"] = seaweed._id.toString()
// item["G"] = seaweed.city
// item["H"] = seaweed.region
// item["I"] = seaweed.name
// }
// println(item)
// }
//
// }
})
@@ -1,82 +1,149 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.HouseValidator
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import net.sf.json.JSONObject
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
ExcelUtils.multiThreadingModify("D:\\商务数据\\建设银行\\副本5市成交案例(测试).xlsx",{List item->
// if(item["R"]){
// return
// }
// def calcDate = null
// if(item["E"]){
// calcDate = item["E"].toString().substring(0, 10)
// }else{
// calcDate = item["B"].toString().substring(0, 10)
// }
// def floor = item["I"].toString().toDouble().toInteger().toString()
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
// def params =[:]
// params.city = item["F"]
// params.district = item["p"]
// params.name = item['Q']
// params.size = item["C"].toString()
// params.floor = floor
// params.maxFloor = maxFloor
// params.direction = dir
// params.decoration = fit
// params.calDate = calcDate
// params.__refresh_cache = "1"
// params.scope = "1"
// JSONObject evaResp = OpenApiClient.evaluate(params)
// if(evaResp.result && evaResp.result.credit > 0.4){
// item["R"] = evaResp.result.totalPrice
// }
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
//school.find(new BasicDBObject("source", "查学区")).each {BasicDBObject item->
// item.category = item.attribute
// item.attribute = null
// school.save(item)
// println(item)
// if(item["H"] != "中粮云景花园"){
// return
//}
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\无经纬度学校-结果.xlsx").each {List item->
// if(item["P"] != null){
// def location = item["P"].split(",") as List
// def data = new BasicDBObject()
// data.map_name = item["N"]
// data.map_region = item["R"]
// data.map_address = item["N"]
// data.map_type = item["S"]
// data.location = [location[0].toString().toDouble(),location[1].toString().toDouble()]
// school.update(new BasicDBObject("_id", new ObjectId(item["A"])), new BasicDBObject('$set', data))
// println(data)
// }
def city = item["F"]
def region = item["G"]
def name = item["H"].toString()
def addr = item["D"]
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property" , false, region, name)
if(resp.name){
item["p"] = resp.district
item['Q'] = resp.name
//}
regionCodeMap = [:]
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
for(def city : cityCodeMap.keySet()){
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
for(def d : datas){
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
}
println(item)
// def calcDate = null
// if(item["E"]){
// calcDate = item["E"].toString().substring(0, 10)
// }else{
// calcDate = item["B"].toString().substring(0, 10)
// }
// def floor = item["I"].toString().toDouble().toInteger().toString()
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
// def params =[:]
// params.city = resp.city
// params.district = resp.district
// params.name = resp.name
// params.size = item["C"].toString()
// params.floor = floor
// params.maxFloor = maxFloor
// params.direction = dir
// params.decoration = fit
// params.calDate = calcDate
// params.scope = "1"
// JSONObject evaResp = OpenApiClient.evaluate(params)
// if(evaResp.result && evaResp.result.credit > 0.4){
// item["R"] = evaResp.result.totalPrice
// }
// println(item)
// }
}
def data = Collections.synchronizedList([])
data << ["ID","CITYID","DISTID","COMMNAME","LNG","LAT","ADDRESS","PS","PS_ADDR","PS_TYPE","PS_SEQNO","PS_LONGITUDE","PS_LATITUDE","MS","MS_ADDR","MS_TYPE","MS_SEQNO","MS_LONGITUDE","MS_LATITUDE","REMARK","CREATETIME","UPDATETIME","SOURCECODE"]
cityCodeMap.keySet().each {def city->
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
if(seaweed.lng){
def item = []
item << seaweed._id.toString()
item << cityCodeMap.get(seaweed.city)
item << regionCodeMap.get(seaweed.city + "-" + seaweed.region)
item << seaweed.name
def lng = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
def lat = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
item << lng
item << lat
item << seaweed.address
BasicDBObject location = new BasicDBObject()
BasicDBObject near = new BasicDBObject()
BasicDBObject geometry = new BasicDBObject("type", "Point")
geometry.append("coordinates", [lng, lat])
near.append('$maxDistance', 1000)
near.append('$geometry', geometry)
location.append('$near', near)
BasicDBObject query = new BasicDBObject()
query.append("location", location)
query.append("attribute",new BasicDBObject('$in', ['省重点', '市重点', '区重点', '县重点', '省级示范']))
query.append("category", new BasicDBObject('$in', ['中学']))
BasicDBObject midSchool = school.findOne(query)
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
// query.append("location", location)
// query.append("attribute", level)
// midSchool = school.findOne(query)
// if(midSchool){
// break
// }
// }
// if(!midSchool){
// query.removeField("attribute")
// midSchool = school.findOne(query)
// }
BasicDBObject primarySchool=null
query.append("category", new BasicDBObject('$in', ['小学']))
primarySchool = school.findOne(query)
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
// query.append("location", location)
// query.append("attribute", level)
// primarySchool = school.findOne(query)
// if(primarySchool){
// break
// }
// }
// if(!primarySchool){
// query.removeField("attribute")
// primarySchool = school.findOne(query)
// }
if(primarySchool){
item << primarySchool.name
item << primarySchool.address
item << primarySchool.attribute
item << null
item << primarySchool.location[0]
item << primarySchool.location[1]
}else {
item << null
item << null
item << null
item << null
item << null
item << null
}
if(midSchool){
item << midSchool.name
item << midSchool.address
item << midSchool.attribute
item << null
item << midSchool.location[0]
item << midSchool.location[1]
}else {
item << null
item << null
item << null
item << null
item << null
item << null
}
item << null
Date c_date = seaweed.c_date
Date m_date = seaweed.m_date
if(!c_date && !m_date){
c_date = DateUtil.add(new Date(), Calendar.YEAR, -1)
}
if(!c_date){
c_date = m_date
}
if(!m_date){
m_date = c_date
}
item <<c_date.format("yyyy-MM-dd HH:mm:ss")
item <<m_date.format("yyyy-MM-dd HH:mm:ss")
item << 'fjw'
println(item)
if(primarySchool || midSchool){
data << item
}
}
})
}
ExcelUtils.write("D:\\无学区房数据.xlsx", data, "结果")
})
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -11,26 +11,44 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
def data = []
def title = ["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg"]
def param_keys = ["tid", "city", "district", "address", "name", "size", "propertyType"]
param_keys.each {
title << "参数_${it}"
}
def resMap_keys = ["city", "district", "name", "avgPrice", "totalPrice", "buildYear"]
resMap_keys.each {
title << "结果_${it}"
}
data << title
def keys = new HashSet()
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if(!result.buildYear || !result.avgPrice){
if (!result.buildYear || !result.avgPrice) {
return
}
def params = JSONObject.fromObject(doc.parameter_map)
keys.addAll(params.keys())
def key = "${params.address}_${params.size}".toString()
if (!hashSet.contains(key)) {
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
param_keys.each {
item << params[it]
}
resMap_keys.each {
item << result[it]
}
data << item
hashSet << key
} else {
@@ -153,55 +153,55 @@ import org.apache.commons.lang3.StringUtils
//}
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
def coords = []
def baidu_points = []
def gaode_points = []
for(int i=0; i < border.size() - 1; i=i+2){
coords << "${border[i]},${border[i+1]}"
if(coords.size() > 50){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
coords.clear()
}
}
if(coords.size() > 0){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
}
dbo.format_city = RegionData.formatCity(dbo.city)
dbo.baidu_border = baidu_points.join(";")
dbo.gaode_border = gaode_points.join(";")
BAIDUMAP_BORDER_COLL.save(dbo)
println(dbo)
}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
// if(seaweed){
// item["A"] = seaweed._id.toString()
// def names = [item['D']]
// if(seaweed.alias){
// names.addAll(seaweed.alias)
//BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
// def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
// def coords = []
// def baidu_points = []
// def gaode_points = []
// for(int i=0; i < border.size() - 1; i=i+2){
// coords << "${border[i]},${border[i+1]}"
// if(coords.size() > 50){
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
// for(JSONObject it : json.result){
// baidu_points << "${it.x},${it.y}"
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
// }
// coords.clear()
// }
// names.unique()
// }
// if(coords.size() > 0){
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
// for(JSONObject it : json.result){
// baidu_points << "${it.x},${it.y}"
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
// }
// }
// dbo.format_city = RegionData.formatCity(dbo.city)
// dbo.baidu_border = baidu_points.join(";")
// dbo.gaode_border = gaode_points.join(";")
// BAIDUMAP_BORDER_COLL.save(dbo)
// println(dbo)
//}
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Ð¡ÇøÈºÏñ²âÊÔÊý¾Ý.xlsx",{List item->
def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
if(seaweed){
// item["A"] = seaweed._id.toString()
def names = [item['D']]
if(seaweed.alias){
names.addAll(seaweed.alias)
}
names.unique()
// if(seaweed.lng){
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
// }
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
// if(one){
// item["I"] = one.baidu_border
// }
// println(item)
// }
//})
def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
if(one){
item["I"] = one.gaode_border
}
println(item)
}
})
@@ -0,0 +1,46 @@
package com.fangjia.script.wenyan.test
import com.aliyuncs.DefaultAcsClient
import com.aliyuncs.IAcsClient
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse
import com.aliyuncs.profile.DefaultProfile
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
def city = item["A"]
def address1 = item["B"]
def address2 = item["C"]
HttpRequest http = new HttpRequest()
def params = new HashMap()
params["city"] = city
params["address_1"] = address1
params["address_2"] = address2
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
JSONObject result = json.result
item["D"] = result?.level
item["E"] = result?.formattedAddress_1
item["F"] = result?.formattedAddress_2
item["G"] = result?.isSameAddress
DefaultProfile profile = DefaultProfile.getProfile(
"cn-hangzhou",
"LTAIdaj4eSO79gzz",
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
IAcsClient client = new DefaultAcsClient(profile);
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
request.setServiceCode("addrp")
request.setAppKey("Xcgtyhupey0h")
request.setText("${address1};${address2}")
GetAddressSimilarityResponse response = client.getAcsResponse(request)
String data = response.getData()
json = JSONObject.fromObject(data)
item["H"] = json["sim_level"]["level"]
item["I"] = json["sim_level"]["address_a"]
item["J"] = json["sim_level"]["address_b"]
item["K"] = json["sim_level"]["is_same"]
println(json)
})
@@ -7,14 +7,14 @@ import java.util.regex.Pattern;
*/
public interface AddressParserPatterns {
Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
Pattern STREET = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?");
Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零|一三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
Pattern STAGE_PATTERN = Pattern.compile("(([零一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([零一二三四五六七八九十百千\\d]+)期");
Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|零|一|二|三|四|五|六|七|八|九|十)+(区|坊)");
Pattern TOWN = Pattern.compile("^.+?(乡|镇|街道)+");
Pattern NUMBER = Pattern.compile("\\d+");
Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
Pattern DISTRICT = Pattern.compile("^.{2,}?(区|县)");
}
@@ -1,13 +1,12 @@
package com.fangjia.service.address;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeoUtils;
import com.fangjia.core.util.StringUtil;
import com.fangjia.core.util.*;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.AmapPoi;
import com.fangjia.dal.mongo.model.bkoff.HouseLoudong;
import com.fangjia.dal.mongo.model.bkoff.Loudong;
import com.fangjia.dal.seaweed.dao.SeaweedDao;
@@ -17,12 +16,14 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors;
import com.fangjia.service.address.processor.AddressProcessors;
import com.fangjia.service.api.AliyunService;
import com.fangjia.service.bo.AliyunBo;
import com.fangjia.service.bo.MatchAddressScore;
import com.fangjia.service.datastore.DistrictService;
import com.fangjia.service.mongo.addrepo.AddressRepoService;
import com.fangjia.service.mongo.bkoff.HouseLoudongService;
import com.fangjia.service.mongo.bkoff.LoudongService;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.bson.types.ObjectId;
@@ -45,7 +46,7 @@ public class AddressParserServiceImpl implements AddressParserService {
private LoudongService loudongService;
private SeaweedDao seaweedDao;
private AliyunService aliyunService;
private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"};
private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "township"};
private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class);
@@ -132,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService {
formatAddress(building);
}
return building;
/*JSONObject result = getDataFromCRF("test");
return new Building();*/
}
@Override
@@ -163,35 +166,188 @@ public class AddressParserServiceImpl implements AddressParserService {
return building;
}
Building matchedBuilding;
//调用CRF算法切分数据
/*JSONObject jsonObj = null;
if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
jsonObj = getDataFromCRF(addr.getOrgAddress());
if(jsonObj != null){
String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") +
(jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") +
(jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : "");
if(StringUtils.isNotBlank(poi)){
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
addr.setKeywords(poi);
}
}
}
}*/
//keywords使用aliyun接口返回的poi
AliyunBo aliyunBo = null;
if(params.isUseAliyunApi()){
if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
aliyunBo = aliyunService.addressStand(params.getAddress(), params.getCity());
if(aliyunBo != null){
String poi = ((aliyunBo.getPoi() == null ? "":aliyunBo.getPoi()) + (aliyunBo.getSubpoi() == null ? "":aliyunBo.getSubpoi())).replaceAll("\\s*", "");
if(StringUtils.isNotBlank(poi)){
addr.setKeywords(poi);
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
addr.setKeywords(poi);
}
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setSuggestCity(aliyunBo.getCity());
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
}
if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){
String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length()));
if(StringUtils.isNotBlank(str)){
String newKeywords = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?"), str);
if(StringUtils.isNotBlank(newKeywords)){
addr.setKeywords(newKeywords);
addr.setParsedPropertyName(newKeywords);
aliyunBo.setRoomno(str.replace(newKeywords,""));
}
}
}
}
}
addr.addKeyword(addr.getOrgAddress());
if(params.isUseMap()){
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP);
}else{
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB);
}
/*if(jsonObj != null){
//乡镇优化
String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : "";
if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){
String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township);
if(StringUtils.isNotBlank(town)){
addr.setTownship(town);
}
}
//道路优化(两边均去掉乡镇)
String street = addr.getStreet();
String streetNumber = addr.getStreetNumber();
String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : "";
String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : "";
//去掉路名中含有乡镇名的
if(StringUtils.isNotBlank(addr.getTownship())){
street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street;
streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf;
}
//crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理
if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){
//Java解析出路名为空
if(StringUtils.isBlank(street)){
street = streetCrf;
if(StringUtils.isNotBlank(streetNumberCrf)){
streetNumber = streetNumberCrf;
}
}else{
//CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则
if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){
street = streetCrf;
}
}
addr.setStreet(street);
addr.setStreetNumber(streetNumber);
}
//poi处理
//没有匹配上库中小区时
if(StringUtils.isBlank(matchedBuilding.getId())){
String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") +
(jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") +
(jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : "");
//去掉路名中含有乡镇名和路名 getParsedPropertyName
if(StringUtils.isNotBlank(addr.getTownship())){
poi = poi.replace(addr.getTownship(),"");
}
if(StringUtils.isNotBlank(addr.getStreet())){
poi = poi.replace(addr.getStreet(),"");
}
addr.setParsedPropertyName(poi);
}
//楼栋、单元、室号
String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", "");
addr.setOrgBuildingNumber(houseNo);
addr.setBuildingNumber(houseNo);
addr.setOrgUnitNumber(cellNo);
addr.setUnitNumber(cellNo);
addr.setFloor(floorNo);
addr.setOrgRoomNumber(roomNo);
addr.setRoomNumber(roomNo);
}*/
matchedBuilding.setProvince(addr.getProvince());
matchedBuilding.setSearchAddress(addr);
matchedBuilding.setSuggestName(getSuggestName(matchedBuilding));
matchedBuilding.setSuggestAddress(getSuggestAddress(matchedBuilding));
if(params.isUseAliyunApi() && aliyunBo != null){
if(params.isUseAliyunApi() && aliyunBo != null && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
completBuildingField(matchedBuilding, aliyunBo, params);
}
Address tempAddr = matchedBuilding.getSearchAddress();
if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
AddressMatchMetaData meta = getMapMeta(addr);
if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){
matchedBuilding.setAddress(meta.getAddress());
}
}
matchedBuilding.setSearchAddress(tempAddr);
return matchedBuilding;
}
private AddressMatchMetaData getMapMeta(Address tempAddr){
tempAddr.addKeyword(tempAddr.getOrgAddress());
List<AddressMatchMetaData> metaDatas = findMatchMetaData(tempAddr, null, MetaDataType.MAP, false);
Map<String,List<AddressMatchMetaData>> metaDataMap = new HashMap<String,List<AddressMatchMetaData>>();
if(StringUtils.isBlank(tempAddr.getDistrict()) && !metaDatas.isEmpty()){
for(AddressMatchMetaData metaData:metaDatas){
if(StringUtils.isNotBlank(metaData.getMapCity())){
List<AddressMatchMetaData> metaList = metaDataMap.containsKey(metaData.getMapCity()) ? metaDataMap.get(metaData.getMapCity()) : new ArrayList<AddressMatchMetaData>();
metaList.add(metaData);
metaDataMap.put(metaData.getMapCity(),metaList);
}
}
String tempCity = "";
if(StringUtils.isNotBlank(tempAddr.getCity()) && metaDataMap.containsKey(tempAddr.getCity())){
tempCity = tempAddr.getCity();
}else{
int size = 0;
for(String key:metaDataMap.keySet()){
if(metaDataMap.get(key).size() > size){
size = metaDataMap.get(key).size();
tempCity = key;
}
}
}
if(StringUtils.isNotBlank(tempCity)){
AddressMatchMetaData meta = metaDataMap.get(tempCity).get(0);
return meta;
}
}
return null;
}
/**
* TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用
* @param address
* @return
*/
/*private JSONObject getDataFromCRF(String address){
HttpRequest request = new HttpRequest();
Map<String,String> params = new HashMap<>();
params.put("address",address);
try {
JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8");
System.out.println(resp);
return resp;
}catch(Exception e){
}
return null;
}*/
/**
* 信息不全时使用阿里接口补全数据
* @param matchedBuilding
@@ -208,28 +364,52 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
Address addr = matchedBuilding.getSearchAddress();
if(StringUtils.isBlank(addr.getProvince())){
if(StringUtils.isNotBlank(aliyunBo.getProv())){
addr.setProvince(formatProvince(aliyunBo.getProv()));
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
}
if(StringUtils.isNotBlank(aliyunBo.getDistrict())){
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
}
addr.setTownship(aliyunBo.getTown());
/*if(StringUtils.isBlank(addr.getProvince())){
addr.setProvince(formatProvince(aliyunBo.getProv()));
}
if(StringUtils.isBlank(addr.getCity())){
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
}
if(StringUtils.isBlank(addr.getDevelopmentZone())){
addr.setDevelopmentZone(aliyunBo.getDevzone());
}
if(StringUtils.isBlank(addr.getDistrict())){
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
}
if(StringUtils.isBlank(addr.getTownship())){
addr.setTownship(aliyunBo.getTown());
}*/
if(StringUtils.isBlank(addr.getDevelopmentZone())){
addr.setDevelopmentZone(aliyunBo.getDevzone());
}
addr.setOrgBuildingNumber(aliyunBo.getHouseno());
/*addr.setOrgBuildingNumber(aliyunBo.getHouseno());
addr.setBuildingNumber(aliyunBo.getHouseno());
addr.setOrgUnitNumber(aliyunBo.getCellno());
addr.setUnitNumber(aliyunBo.getCellno());
addr.setOrgRoomNumber(aliyunBo.getRoomno());
addr.setRoomNumber(aliyunBo.getRoomno());
addr.setUnitNumber(aliyunBo.getCellno());*/
/*if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){
addr.setOrgRoomNumber(aliyunBo.getRoomno());
addr.setRoomNumber(aliyunBo.getRoomno());
}else{
addr.setRoomNumber(null);
addr.setRoomNumber(null);
}*/
/*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", "");
String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", "");
if(orgRoom!=null && orgRoom.equals(aliRoom)){
addr.setRoomNumber(aliyunBo.getRoomno());
}else{
addr.setRoomNumber(null);
}*/
if(StringUtils.isBlank(addr.getStreet())){
addr.setStreet(aliyunBo.getRoad());
@@ -318,6 +498,7 @@ public class AddressParserServiceImpl implements AddressParserService {
addr.setSuggestDistrict(params.getSuggestDistrict());
addr.setSuggestName(params.getSuggestName());
addr.setPreciseLevel(params.getPreciseLevel());
addr.setAddressParserGoal(params.getAddressParserGoal());
addr.setProvince(params.getProvince());
addr.setDebugName(params.getDebugName());
return AddressProcessors.process(addr);
@@ -362,7 +543,7 @@ public class AddressParserServiceImpl implements AddressParserService {
* @param metaDataType
* @return
*/
private List<AddressMatchMetaData> findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType){
private List<AddressMatchMetaData> findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType, boolean validCity){
if(MetaDataType.MAP == metaDataType){
List<String> keywords = address.getKeywordList();
if(keywords.isEmpty()){
@@ -375,7 +556,7 @@ public class AddressParserServiceImpl implements AddressParserService {
}
keywords.add(standardAddress);
}
return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100);
return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100, validCity);
}else{
return SEAWEED_LOCAL_CACHE.get(new MetaDataCacheKey(address.getCity(), buildingType, MetaDataType.CPDB));
}
@@ -517,20 +698,20 @@ public class AddressParserServiceImpl implements AddressParserService {
scoreResult.setOrgAddress(addr.getOrgAddress());
if(buildingType != null) {
if(MetaDataType.MAP == metaDataType){
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP);
metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB));
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP, true);
metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true));
}else{
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB);
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true);
}
} else {
if(MetaDataType.MAP == metaDataType){
metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP);
metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP, true);
}
for (BuildingType bt : addr.guessBuildType()) {
if(metaDatas == null){
metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB);
metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB, true);
}else{
metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB));
metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB, true));
}
}
}
@@ -960,15 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService {
if(StringUtils.isNotBlank(address.getTownship())){
formattedAddressBuilder.append(address.getTownship());
}
if(StringUtils.isNotBlank(building.getAddress())){
Address orgAddr = new Address();
orgAddr.setOrgAddress(building.getAddress());
Address addr = AddressProcessors.process(orgAddr);
if(StringUtils.isBlank(address.getStreet())){
address.setStreet(addr.getStreet());
}
if(StringUtils.isBlank(address.getStreetNumber())){
address.setStreetNumber(addr.getStreetNumber());
}
}
if(StringUtils.isNotBlank(address.getStreet())){
formattedAddressBuilder.append(address.getStreet());
}
if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){
formattedAddressBuilder.append(address.getStreetNumber());
}
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
formattedAddressBuilder.append(building.getAddress());
}
if(StringUtils.isNotBlank(building.getSuggestName())){
formattedAddressBuilder.append(building.getSuggestName());
}
@@ -985,18 +1175,18 @@ public class AddressParserServiceImpl implements AddressParserService {
&& address.getStreetNumber().endsWith(address.getOrgBuildingNumber())){
}else{
formattedAddressBuilder.append(address.getBuildingNumber());
// formattedAddressBuilder.append(address.getBuildingNumber() + "¶°");
// formattedAddressBuilder.append(address.getBuildingNumber());
formattedAddressBuilder.append(address.getBuildingNumber() + "栋");
}
}
if(StringUtils.isNotBlank(address.getOrgUnitNumber())){
formattedAddressBuilder.append(address.getUnitNumber());
// formattedAddressBuilder.append(address.getUnitNumber() + "µ¥Ôª");
// formattedAddressBuilder.append(address.getUnitNumber());
formattedAddressBuilder.append(address.getUnitNumber() + "单元");
}
if(StringUtils.isNotBlank(address.getOrgRoomNumber())){
formattedAddressBuilder.append(address.getRoomNumber());
// formattedAddressBuilder.append(address.getRoomNumber() + "ÊÒ");
if(StringUtils.isNotBlank(address.getRoomNumber())){
// formattedAddressBuilder.append(address.getRoomNumber());
formattedAddressBuilder.append(address.getRoomNumber() + "室");
}
address.setFormattedAddress(formattedAddressBuilder.toString());
@@ -63,14 +63,18 @@ public class AddressFormatterFactory {
});
/*人民中路123-3501 -->人民中路123号3501室*/
formatters.add(new AbstractAddressFormatter(Pattern.compile(".*[路|弄]\\d+-\\d{3,4}$")) {
/*人民中路123-128 -->人民中路123号-3501号*/
formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?\\d+-\\d{3,4}(号)?$")) {
final Pattern TARGET_PATTERN = Pattern.compile("(?<=(路|号))\\d+-\\d{3,4}$");
final Pattern TARGET_PATTERN = Pattern.compile("(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?\\d+-\\d{3,4}(号)?$");
@Override
protected String doFormat(String address) {
String targetStr = DataUtil.match(address, TARGET_PATTERN);
return address.replace(targetStr, targetStr.replace("-", "号") + "室");
if(targetStr.endsWith("号")){
return address.replace(targetStr, targetStr.replace("-", "号-"));
}else{
return address.replace(targetStr, targetStr.replace("-", "号-") + "号");
}
}
});
@@ -259,7 +263,38 @@ public class AddressFormatterFactory {
@Override
protected String doFormat(String address) {
String matchStr = DataUtil.match(address, TARGET);
String formatStr = matchStr + "号";
String formatStr = matchStr.replaceAll("段-","段");
if(!matchStr.endsWith("号")){
formatStr += "号";
}
return address.replace(matchStr,formatStr);
}
});
/**
* 剑南路东段-12号-18号
*/
formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))")) {
final Pattern TARGET = Pattern.compile("(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))");
@Override
protected String doFormat(String address) {
String matchStr = DataUtil.match(address, TARGET);
String formatStr = matchStr;
if(!matchStr.endsWith("号")){
formatStr += "号";
}
return address.replace(matchStr,formatStr);
}
});
/**
* 剑南路东段-12号-18号
*/
formatters.add(new AbstractAddressFormatter(Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))")) {
final Pattern TARGET = Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))");
@Override
protected String doFormat(String address) {
String matchStr = DataUtil.match(address, TARGET);
String formatStr = matchStr.replace("-","");
return address.replace(matchStr,formatStr);
}
});
@@ -77,7 +77,27 @@ public class Address {
private String debugName;
private AliyunBo aliyunBo;
//使用CRF切分的地址
private String crfAddress;
public String getCrfAddress() {
return crfAddress;
}
public void setCrfAddress(String crfAddress) {
this.crfAddress = crfAddress;
}
public AddressParserGoal getAddressParserGoal() {
return addressParserGoal;
}
public void setAddressParserGoal(AddressParserGoal addressParserGoal) {
this.addressParserGoal = addressParserGoal;
}
/**地址解析目的*/
private AddressParserGoal addressParserGoal;
public AliyunBo getAliyunBo() {
return aliyunBo;
}
@@ -44,6 +44,24 @@ public class AddressMatchMetaData {
private Double lat;
private String map;
public String getMapProvince() {
return mapProvince;
}
public void setMapProvince(String mapProvince) {
this.mapProvince = mapProvince;
}
public String getMapCity() {
return mapCity;
}
public void setMapCity(String mapCity) {
this.mapCity = mapCity;
}
private String mapProvince;
private String mapCity;
public void addName(String name, float priority){
if(StringUtils.isNotBlank(name)){
String formatName = AddressParserUtil.standardAddressAndName(name);
@@ -64,9 +64,7 @@ public class Building {
this.alias = d.getAlias();
this.addrAlias = d.getAddrAlias();
this.propertyType = d.getEstateType2();
if(StringUtils.isNotBlank(d.getStreetCommunity())){
this.township = d.getStreetCommunity();
}
this.township = d.getTownship();
if(d.getLat() != null && d.getLng() != null){
this.lat = d.getLat()/10000000000d;
this.lng = d.getLng()/10000000000d;
@@ -90,9 +88,7 @@ public class Building {
this.lat = d.getLat2()/10000000000d;
this.lng = d.getLng2()/10000000000d;
}
if(StringUtils.isNotBlank(d.getStreetCommunity())){
this.township = d.getStreetCommunity();
}
this.township = d.getTownship();
}
public Building() {
@@ -25,7 +25,15 @@ public class MatchParams {
private boolean useMap;
/**是否使用阿里云接口*/
private boolean useAliyunApi;
//是否使用CRF算法
private boolean useCrf;
public boolean isUseCrf() {
return useCrf;
}
public void setUseCrf(boolean useCrf) {
this.useCrf = useCrf;
}
public boolean isContainsLocation() {
return containsLocation;
}
@@ -50,18 +50,23 @@ public class CityProcessor extends AbstractAddressProcessor {
return RepeatStatus.NEXT;
}
}
if(StringUtils.isNotBlank(city)){
if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){
String formatCity = RegionData.formatCity(city);
String possibleCity = null;
String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null;
String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity;
/**如果省份和城市信息相互矛盾 则用区域推断城市*/
if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){
List<String> possibleCities = speculateCityNameByRegion(address, city);
if(isUnique(possibleCities)){
formatCity = getFirst(possibleCities);
confirmCity = formatCity;
}
}
if(null != formatCity){
city = formatCity;
if(AddressParserGoal.Match.equals(address.getAddressParserGoal()) && (formatCity == null || (possibleCity!=null && !formatCity.equals(possibleCity)))){
confirmCity = possibleCity;
}
if(null != confirmCity){
city = confirmCity;
address.setCity(city);
address.setCityStatus(CityStatus.SURE);
}
@@ -98,7 +103,7 @@ public class CityProcessor extends AbstractAddressProcessor {
address.setCity(city);
}
if(StringUtils.isNotBlank(city) && scoreRegion(city, orgAddress) > 0.4){
orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?" , "");
orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?(?!(路|街|大道|巷|道|段))" , "");
address.setAddress(orgAddress);
address.setCityStatus(CityStatus.SURE);
return RepeatStatus.NEXT;
@@ -166,7 +171,7 @@ public class CityProcessor extends AbstractAddressProcessor {
String street = getStreet(address);
for(String city : allCity){
if(street != null){
MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)"), street);
MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street);
if(matchResult !=null){
address = address.replace(matchResult.matchContent, "");
}
@@ -199,6 +204,10 @@ public class CityProcessor extends AbstractAddressProcessor {
}
Map<String, List<String>> regionMap = RegionData.getAllRegions(city);
for(String region : regionMap.keySet()){
MatchResult matchResult = matchFirst(Pattern.compile(region + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street);
if(matchResult !=null){
address = address.replace(matchResult.matchContent, "");
}
if(address.indexOf(region, index) > -1){
if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){
weight.score += 2;
@@ -2,6 +2,7 @@ package com.fangjia.service.address.processor;
import com.fangjia.core.RegionData;
import com.fangjia.service.address.mode.Address;
import com.fangjia.service.address.mode.AddressParserGoal;
import com.fangjia.service.address.mode.CityStatus;
import com.fangjia.service.address.mode.ProvinceData;
import org.apache.commons.lang.StringUtils;
@@ -24,7 +25,7 @@ public class ProvinceProcessor extends AbstractAddressProcessor {
String orgAddress = address.getAddress();
String[] provinceInfo = getProvince(orgAddress);
if(provinceInfo == null && address.getCity() != null){
if(provinceInfo == null && address.getCity() != null && !AddressParserGoal.Match.equals(address.getAddressParserGoal())){
provinceInfo = getProvince(address.getCity());
}
if (null != provinceInfo) {
@@ -18,6 +18,7 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
static final Pattern NUMBER_END = Pattern.compile("(?<=\\D)\\d{3,4}$");
static final Pattern HOUSE_NUMBER = Pattern.compile("\\d{3,4}");
static final String HOUSE_UNITS_PATTERN = "(ÊÒ|·¿|ºÅ·¿|ºÅ(?!Ôº)|»§|ÆÌ)+";
static final Pattern HOUSE_PATTERN_CASE02 = Pattern.compile("(\\d+(ÊÒ|·¿|ºÅ)?[¡¢]?)+$");
@Override
public RepeatStatus process(Address address) {
@@ -27,10 +28,17 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end);
address.setOrgRoomNumber(matchResult.matchContent);
address.setAddress(orgAddress);
analyFloorUseRoomNumber(address);
// analyFloorUseRoomNumber(address);
return RepeatStatus.NEXT;
}
MatchResult matchResult2 = matchFirst(HOUSE_PATTERN_CASE02, orgAddress);
if (matchResult2 != null) {
orgAddress = removeSubstring(orgAddress, matchResult2.start, matchResult2.end);
address.setOrgRoomNumber(matchResult2.matchContent);
address.setAddress(orgAddress);
// analyFloorUseRoomNumber(address);
return RepeatStatus.NEXT;
}
matchResult = matchFirst(HOUSE_PATTERN, orgAddress);
if (matchResult != null) {
orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end);
@@ -39,10 +47,10 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
} else {
address.setOrgRoomNumber(matchResult.matchContent + "ÊÒ");
}
analyFloorUseRoomNumber(address);
// analyFloorUseRoomNumber(address);
address.setAddress(orgAddress);
}
analyFloorUseRoomNumber(address);
// analyFloorUseRoomNumber(address);
removeHouseUnit(address);
return RepeatStatus.NEXT;
}
@@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor {
@Override
public RepeatStatus process(Address address) {
String orgAddr = address.getEffectiveAddress();
address.setCrfAddress(address.getAddress());
MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress());
if (matchResult != null) {
String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent);
@@ -41,6 +41,7 @@ public class TooltipProcessor extends AbstractAddressProcessor {
offset += matchResult.matchContent.length();
if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){
address.setTooltip(tooltip);
address.setStreet(getStreet(tooltip));
} else if(containsDistrictName(tooltip)){
address.setTooltip(tooltip);
} else if(RegionData.isRegion(address.getCity(), tooltip)){
@@ -1,6 +1,7 @@
package com.fangjia.service.address.processor;
import com.fangjia.service.address.mode.Address;
import com.fangjia.service.cnAnalyzer.Token;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
@@ -14,6 +15,15 @@ public class VillageNameProcessor extends AbstractAddressProcessor {
@Override
public AbstractAddressProcessor.RepeatStatus process(Address address) {
String orgAddress = address.getAddress();
if(address.getTokens()!=null){
for(Token token:address.getTokens()){
if("district".equals(token.getKeywordTypes())){
address.setKeywords(token.getName());
address.setAddress(orgAddress.replace(token.getName(),""));
return RepeatStatus.NEXT;
}
}
}
List<String> names = tokenDistrictNames(orgAddress);
List<String> kws = new ArrayList<String>();
for(String name : names){
@@ -1,5 +1,7 @@
package com.fangjia.service.api;
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest;
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse;
import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest;
import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse;
import com.fangjia.core.util.JsonUtils;
@@ -13,6 +15,9 @@ import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import java.util.ArrayList;
import java.util.List;
public class AliyunServiceImpl implements AliyunService {
protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass());
@@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService {
if(StringUtils.isNotBlank(city)){
request.setDefaultCity(city);
}
AliyunBo aliyunBo = new AliyunBo();
try{
StructureAddressResponse response = client.getAcsResponse(request);
JSONObject responseJson = JSONObject.fromObject(response.getData());
String value = (String) responseJson.get("structure");
AliyunBo aliyunBo = new AliyunBo();
for (String key:value.split("\t")){
String[] arry = key.split("=");
wrapField(arry[0],arry[1], aliyunBo);
}
return aliyunBo;
}catch(Exception e){
LOGGER.error("", e);
return null;
LOGGER.error("调用阿里标准化接口失败:", e);
}
try{
GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest();
divisionRequest.setServiceCode("addrp");
divisionRequest.setAppKey(APPKEY);
divisionRequest.setText(address);
GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest);
JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData());
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
String[] array = value.replace("\t", ";").split(";");
if(array.length > 4){
List<String> tmp = new ArrayList<String>();
for(String str:array){
if(str!=null && str.length()!=0){
tmp.add(str);
}
}
array= tmp.toArray(new String[4]);
}
aliyunBo.setProv(array[0]);
aliyunBo.setCity(array[1]);
aliyunBo.setDistrict(array[2]);
aliyunBo.setTown(array[3]);
}catch(Exception e){
LOGGER.error("调用阿里行政区划识别接口失败", e);
}
return aliyunBo;
}
private void wrapField(String field,String value, AliyunBo aliyunBo){
@@ -81,26 +110,59 @@ public class AliyunServiceImpl implements AliyunService {
}
}
public static void main(String[] args) throws ClientException {
DefaultProfile profile = DefaultProfile.getProfile(
String value = "";
String[] array = value.replace("\t", ";").split(";");
if(array.length > 4){
List<String> tmp = new ArrayList<String>();
for(String str:array){
if(str!=null && str.length()!=0){
tmp.add(str);
}
}
array= tmp.toArray(new String[4]);
}
System.out.println(array[0]);
System.out.println(array[1]);
System.out.println(array[2]);
System.out.println(array[3]);
/*DefaultProfile profile = DefaultProfile.getProfile(
REGIONID,
ACCESSKEYID,
SECRET);
IAcsClient client = new DefaultAcsClient(profile);
StructureAddressRequest request = new StructureAddressRequest();
// StructureAddressRequest request = new StructureAddressRequest();
GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest();
request.setServiceCode("addrp");
request.setAppKey(APPKEY);
request.setText("杭州市文一西路969号");
request.setDefaultProvince("浙江省");
request.setDefaultCity("杭州市");
request.setDefaultDistrict("余杭区");
StructureAddressResponse response = client.getAcsResponse(request);
request.setText("天青路260号");
// request.setDefaultProvince("浙江省");
// request.setDefaultCity("杭州市");
// request.setDefaultDistrict("余杭区");
// StructureAddressResponse response = client.getAcsResponse(request);
GetAddressDivisionCodeResponse response = client.getAcsResponse(request);
JSONObject responseJson = JSONObject.fromObject(response.getData());
String value = (String) responseJson.get("structure");
JSONObject json = new JSONObject();
System.out.println(responseJson);
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
String[] array = value.replace("\t", ";").split(";");
if(array.length > 4){
List<String> tmp = new ArrayList<String>();
for(String str:array){
if(str!=null && str.length()!=0){
tmp.add(str);
}
}
array= tmp.toArray(new String[4]);
}
System.out.println(array[0]);
System.out.println(array[1]);
System.out.println(array[2]);
System.out.println(array[3]);*/
/*JSONObject json = new JSONObject();
for (String key:value.split("\t")){
String[] arry = key.split("=");
json.put(arry[0],arry[1]);
}
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/
}
}
@@ -68,7 +68,7 @@ public interface AmapPoiService {
* @param types
* @return
*/
List<AmapPoi> search(String city, String keywords, Integer limit, String types);
List<AmapPoi> search(String city, String keywords, Integer limit, String types, boolean validCity);
/**
@@ -1,5 +1,6 @@
package com.fangjia.service.api;
import com.fangjia.core.ProvinceData;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeoUtils;
@@ -258,14 +259,14 @@ public class AmapPoiServiceImpl implements AmapPoiService {
}
@Override
public List<AmapPoi> search(String city, String keywords, Integer limit, String types) {
public List<AmapPoi> search(String city, String keywords, Integer limit, String types, boolean validCity) {
Map<String, String> params = new HashMap<String, String>();
String area = city;
params.put("city", area);
params.put("output", "json");
params.put("keywords", keywords);
params.put("key", "4ccfe3c898a9489884bd2bcb1ef6dcde");
params.put("citylimit", "true");
// params.put("citylimit", "true");
params.put("offset", limit == null?"25":String.valueOf(limit));
if(types != null){
params.put("types", "120301&120302&120201&060101");
@@ -351,15 +352,17 @@ public class AmapPoiServiceImpl implements AmapPoiService {
cityname = poi.getString("cityname");
}
cityname = RegionData.formatCity(cityname);
if(!city.equals(cityname)){
if((city!=null && !city.equals(cityname) && validCity) || cityname == null){
continue;
}
pname = ProvinceData.formatProvince(pname);
AmapPoi amapPoi = new AmapPoi();
amapPoi.setCity(city);
amapPoi.setCity(cityname);
amapPoi.setPname(pname);
if(poi.containsKey("adname") && poi.get("adname") instanceof String){
adname = poi.getString("adname");
String region = RegionData.formatRegion(city, adname);
if(RegionData.isRegion(city, region)){
String region = RegionData.formatRegion(cityname, adname);
if(RegionData.isRegion(cityname, region)){
amapPoi.setRegion(region);
}
}
@@ -380,8 +383,8 @@ public class AmapPoiServiceImpl implements AmapPoiService {
amapPoi.setAlias(alias);
}
if(poi.containsKey("adname") && poi.get("adname") instanceof String){
String region = RegionData.formatRegion(city, poi.getString("adname"));
if(RegionData.isRegion(city, region)){
String region = RegionData.formatRegion(cityname, poi.getString("adname"));
if(RegionData.isRegion(cityname, region)){
amapPoi.setRegion(region);
}
}
@@ -391,7 +394,7 @@ public class AmapPoiServiceImpl implements AmapPoiService {
.replace(cityname, "").replace(adname, "");
amapPoi.setAddress(AddressParserUtil.optimizeAddress(address));
}
amapPoi.setCity(city);
amapPoi.setCity(cityname);
if(poi.containsKey("location") && poi.get("location") instanceof String){
String[] location = poi.getString("location").split(",");
amapPoi.setLng(Double.valueOf(location[0]));
@@ -2495,10 +2495,12 @@ public class DistrictServiceImpl implements DistrictService {
params.add(new QueryParam("visible", 0));
params.add(new QueryParam("city", city));
params.add(new QueryParam("category", "district"));
params.add(new QueryParam("keyword", keyword));
Query query = boostQueryDistrictPolicy.create(ServiceUtil.removeAndGetKeyword(params));
if (query != null) {
params.add(new QueryParam(QueryParam.SR_PARAM_NAME, new SrQuery(query)));
if (StringUtils.isNotBlank(keyword)) {
BooleanQuery query = new BooleanQuery();
Query q1 = new TermQuery(new Term("keyword", keyword));
q1.setBoost(0.99f);
query.add(q1, Occur.SHOULD);
params.add(new QueryParam("$sr", new SrQuery(query)));
}
return districtDao.findAllByParams(params, limit);
}
@@ -37,7 +37,7 @@ public interface AddressRepoService {
* @param max
* @return
*/
List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max);
List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max, boolean validCity);
/**
*
@@ -87,7 +87,7 @@ public class AddressRepoServiceImpl implements AddressRepoService {
return addrs;
}
public List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max){
public List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max, boolean validCity){
String types = null;
String tag = null;
String category = null;
@@ -104,7 +104,11 @@ public class AddressRepoServiceImpl implements AddressRepoService {
tag = "°Ù»õÉ̳¡";
category = "×ÛºÏÉ̳¡";
}
List<AmapPoi> pois = amapPoiService.search(city, StringUtils.join(keywords, "|"), 20, types);
List<AmapPoi> pois = new ArrayList<AmapPoi>();
for(String keyword:keywords){
List<AmapPoi> poiList = amapPoiService.search(city, keyword, 20, types, validCity);
pois.addAll(poiList);
}
List<AddressMatchMetaData> matchMetaDatas = new ArrayList<AddressMatchMetaData>(pois.size());
for(AmapPoi poi: pois){
AddressMatchMetaData data = new AddressMatchMetaData();
@@ -121,6 +125,8 @@ public class AddressRepoServiceImpl implements AddressRepoService {
data.setType(poi.getType());
data.setMap("¸ßµÂ");
data.setCity(city);
data.setMapCity(poi.getCity());
data.setMapProvince(poi.getPname());
if(poi.getAlias() != null){
for(String alia : poi.getAlias()){
data.addName(alia, MatchText.ALIAS_PRIORITY);
@@ -284,5 +284,12 @@ public interface AddressService {
* @return
*/
AddressCheckTruth aroundConsumeGrade(String city, String address);
/**
* 量地址相似度评级
* @param result1
* @param result2
* @return 返回是否同一地址、输出级别(哪一级),相似度得分
*/
Map<String,Object> addressSimilarity(String result1, String result2);
}
@@ -5,18 +5,7 @@ import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URLEncoder;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -25,6 +14,9 @@ import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.fangjia.core.util.*;
import com.fangjia.service.address.AddressParserUtil;
import com.fangjia.service.address.mode.MatchResult;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
@@ -49,18 +41,8 @@ import org.sothis.core.util.Closure;
import com.fangjia.core.BlockData;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.AddressParser;
import com.fangjia.core.util.AddressParserUtils;
import com.fangjia.core.util.AddressResult;
import com.fangjia.core.util.BeanUtils;
import com.fangjia.core.util.CacheUtils;
import com.fangjia.core.util.CacheUtils.CachePolicyEnum;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.DateUtil;
import com.fangjia.core.util.GeoUtils;
import com.fangjia.core.util.GeoUtils.GaussSphere;
import com.fangjia.core.util.HttpRequest;
import com.fangjia.core.util.LoggerFactory;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.SdQuery;
import com.fangjia.dal.SrQuery;
@@ -5074,7 +5056,186 @@ public class AddressServiceImpl implements AddressService {
}
return checkTr;
}
@Override
public Map<String, Object> addressSimilarity(String result1, String result2) {
JSONObject json1 = (JSONObject) JSONObject.fromObject(result1).get("result");
JSONObject json2 = (JSONObject) JSONObject.fromObject(result2).get("result");
Map<String,Object> resultMap = new HashMap<String, Object>();
//判断是否同一地址
boolean sameAddress = validSameAddress(json1, json2);
String formatAddress1_pinyin = AddressParserUtil.toHanYuPinYin(getFromJson(json1, "formattedAddress"));
String formatAddress2_pinyin = AddressParserUtil.toHanYuPinYin(getFromJson(json2,"formattedAddress"));
double similar = AddressParserUtil.getSimilarityRatio(formatAddress1_pinyin, formatAddress2_pinyin);
String level = getAddressSimilarityLevel(json1, json2);
resultMap.put("isSameAddress", sameAddress);
// resultMap.put("score", similar);
resultMap.put("level", level);
resultMap.put("formattedAddress_1", getFromJson(json1,"formattedAddress"));
resultMap.put("formattedAddress_2", getFromJson(json2,"formattedAddress"));
return resultMap;
}
private boolean validSameAddress(JSONObject json1, JSONObject json2){
//标准化后地址完全一样
if(getFromJson(json1, "formattedAddress").equals(getFromJson(json2, "formattedAddress"))){
return true;
}
//判断判断省、市、区、乡镇、楼栋、单元、室号是否相同
if(!verifySameField(getFromJson(json1,"province"), getFromJson(json2,"province"), false)
|| !verifySameField(getFromJson(json1,"city"), getFromJson(json2,"city"), false)
|| !verifySameField(getFromJson(json1,"district"), getFromJson(json2,"district"), false)
|| !verifySameField(getFromJson(json1,"township"), getFromJson(json2,"township"), false)
|| !verifySameField(getFromJson(json1,"buildingNumber"), getFromJson(json2,"buildingNumber"), true)
|| !verifySameField(getFromJson(json1,"unitNumber"), getFromJson(json2,"unitNumber"), true)
|| !verifySameField(getFromJson(json1,"roomNumber"), getFromJson(json2,"roomNumber"), true)){
return false;
}
String street1 = (StringUtils.isNotBlank(getFromJson(json1,"street")) ? getFromJson(json1,"street") : "") + (StringUtils.isNotBlank(getFromJson(json1,"streetNumber")) ? getFromJson(json1,"streetNumber") : "");
String street2 = (StringUtils.isNotBlank(getFromJson(json2,"street")) ? getFromJson(json2,"street") : "") + (StringUtils.isNotBlank(getFromJson(json2,"streetNumber")) ? getFromJson(json2,"streetNumber") : "");
String name1 = StringUtils.isNotBlank(getFromJson(json1,"name")) ? getFromJson(json1,"name") : "";
String name2 = StringUtils.isNotBlank(getFromJson(json2,"name")) ? getFromJson(json2,"name") : "";
//两个地址中路名+路号和poi全部为空时返回true
if(StringUtils.isBlank(street1) && StringUtils.isBlank(street2) && StringUtils.isBlank(name1) && StringUtils.isBlank(name2)){
return true;
}
//同一条地址里路名和poi不能全为空而另一条至少有一项有值
if((StringUtils.isBlank(street1) && StringUtils.isBlank(name1)) || (StringUtils.isBlank(street2) && StringUtils.isBlank(name2))){
return false;
}
//poi有任意一个为空则路名、路号必须完全一致
if(StringUtils.isBlank(name1) || StringUtils.isBlank(name2)){
if(StringUtils.isNotBlank(getFromJson(json1,"street")) && getFromJson(json1,"street").equals(getFromJson(json2,"street"))
&& StringUtils.isNotBlank(getFromJson(json1,"streetNumber")) && getFromJson(json1,"streetNumber").equals(getFromJson(json2,"streetNumber"))){
return true;
}else{
return false;
}
}
//poi全不为空时,路名+路号一致(可为空),poi处理后一致
if(verifySameField(street1, street2, false) && verifySamePoi(name1, name2)){
return true;
}
return false;
}
private String getFromJson(JSONObject json, String key){
if(json.containsKey(key)){
return json.getString(key);
}
return null;
}
/**
* 获取两地址的相似度评级(相同数据的最小颗粒度)
* @return
*/
private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){
String level = null;
LinkedList<String> list = new LinkedList<String>(Arrays.asList("province", "city", "district", "township", "street", "streetNumber"
, "name", "buildingNumber", "unitNumber", "roomNumber"));
for(String key:list){
if(continueJudgeValue(json1, json2, key) == 3){
return level;
}else{
level = continueJudgeValue(json1, json2, key) == 1 ? key : (level == null ? null : level);
}
}
return level;
}
/**
* 判断level时是否继续往下判断
* @param json1
* @param json2
* @param key
* @return 1 继续判断并且level有值 2继续判断level没值 3 停止
*/
private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){
if("buildingNumber".equals(key) || "unitNumber".equals(key) || "roomNumber".equals(key)){
if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))
&& getFromJson(json1,key).equals(getFromJson(json2,key))){
return 1;
}else{
return 3;
}
}
if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){
if("name".equals(key)){
return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3;
}else{
if(getFromJson(json1,key).equals(getFromJson(json2,key))){
return 1;
}else{
return 3;
}
}
}
return 2;
}
/**
* 判断省、市、区、乡镇、楼栋、单元、室号是否相同(两者全为空时返回true)
* @param str1
* @param str2
* @param flag true时严格判断 两者完全一致返回true(全为空或者全部有值并且值一致)
* false时 两者完全一致或一方为空返回true(全为空或者全部有值并且值一致或任一方值为空)
* @return
*/
private boolean verifySameField(String str1, String str2,boolean flag){
if(flag){
if(StringUtils.isBlank(str1) && StringUtils.isBlank(str2)){
return true;
}
return StringUtils.isNotBlank(str1) ? str1.equals(str2) : false;
}else{
if(StringUtils.isBlank(str1) || StringUtils.isBlank(str2)){
return true;
}
return str1.equals(str2);
}
}
/**
* //判断是否同一个poi(全部有数据时)
* @param name1
* @param name2
* @return
*/
private boolean verifySamePoi(String name1, String name2){
if(StringUtils.isBlank(name1) && StringUtils.isBlank(name2)){
return false;
}
if(name1.equals(name2)){
return true;
}
//分期
String name1_stage = AddressParserUtil.match(AddressParserPatterns.STAGE_PATTERN,name1);
String name2_stage = AddressParserUtil.match(AddressParserPatterns.STAGE_PATTERN,name2);
//分区
String name1_partition = AddressParserUtil.match(AddressParserPatterns.PARTITION_PATTERN,name1);
String name2_partition = AddressParserUtil.match(AddressParserPatterns.PARTITION_PATTERN,name2);
//数字
String name1_number = AddressParserUtil.match(Pattern.compile("[\\d一二三四五六七八九十千百万]+"),name1);
String name2_number = AddressParserUtil.match(Pattern.compile("[\\d一二三四五六七八九十千百万]+"),name2);
if(!verifySameField(name1_stage, name2_stage, true) || !verifySameField(name1_partition, name2_partition, true)|| !verifySameField(name1_number, name2_number, true)){
return false;
}
return isSameName(name1, name2);
}
private boolean isSameName(String name1, String name2){
String name_1_pinyin = AddressParserUtil.toHanYuPinYin(name1);
String name_2_pinyin = AddressParserUtil.toHanYuPinYin(name2);
//拼音或文本数据包含并且
boolean pinYinContains = (name_1_pinyin.contains(name_2_pinyin) || name_2_pinyin.contains(name_1_pinyin));
if(pinYinContains){
double similar = AddressParserUtil.getSimilarityRatio(name_1_pinyin, name_2_pinyin);
if(similar > 0.5){
return true;
}
}
return false;
}
/**
* ÉèÖÃÇøÓòÏû·ÑÖ¸Êý
* @param checkTr
+1 -1
View File
@@ -29,7 +29,7 @@
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-core</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.0</version>
</dependency>
<dependency>
<groupId>com.fangjia</groupId>
+2
View File
@@ -0,0 +1,2 @@
FROM 192.168.0.19:9012/tomcat6:latest
COPY target/*.war /home/goojia/webapps/ROOT.war
+1 -1
View File
@@ -51,7 +51,7 @@
<dependency>
<groupId>org.sothis</groupId>
<artifactId>sothis-web</artifactId>
<version>0.2.0-SNAPSHOT</version>
<version>0.2.0</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
@@ -22,14 +22,8 @@ public class DistrictController {
private DistrictService districtService;
public Object loadDistrictAction(DistrictModel model, HttpServletRequest request){
String url = openSearchService.createQueryUrl(model.getCityName(), model.getKeywords(), false);
String content = openSearchService.findResultByUrl(url);
List<String> regions = districtService.getRegionByCity(model.getCityName());
Map<String, List<DistrictBo>> map = openSearchService.parseResult(content, regions, model.getKeywords());
if(map.size() == 0){
List<District> districts = districtService.findNewDistrict(model.getCityName(), model.getKeywords());
map = openSearchService.createResult(districts);
}
List<District> districts = districtService.findDistrictByKeyword(model.getCityName(), model.getKeywords(), 10);
Map<String, List<DistrictBo>> map = openSearchService.createResult(districts);
JsonModel ret = new JsonModel();
ret.setJsonModel(map);
return ret;
@@ -155,11 +155,11 @@ public class BaseModel extends ModelAndViewSupport {
}
public final String getRoot() {
return (CONFIG.isDevelopmentEnvironment() ? "/geluke" : "");
return "http://geluke.fangjia.com";
}
public final String getWwwUrl() {
return WWW_URL;
return "http://geluke.fangjia.com";
}
public final Date getNow() {
@@ -270,7 +270,7 @@ public class BaseModel extends ModelAndViewSupport {
}
public final String getCityUrl() {
return CONFIG.getCityDomainName() + getRoot();
return "http://geluke.fangjia.com";
}
public final String getDomain() {
@@ -17,16 +17,32 @@ fangjia.auto.url=${fangjia.auto.host}:${fangjia.auto.port}/
fangjia.file.mongo.host=fs.mongo.db.fangjia.com
fangjia.file.mongo.port=27017
#fangjia.log.mongo.host=log.mongo.db.fangjia.com
#fangjia.log.mongo.port=27017
fangjia.log.mongo.host=log.mongo.db.fangjia.com
fangjia.log.mongo.port=27017
fangjia.log.mongo.port=52002
fangjia.log.mongodb.mongoUri=mongodb://${fangjia.log.mongo.host}:${fangjia.log.mongo.port}/
fangjia.mongo.host=mongo.db.fangjia.com
fangjia.mongo.port=27017
fangjia.mongo.port=28017
fangjia.secondary.mongo.host=secondary.mongo.db.fangjia.com
fangjia.secondary.mongo.port=28017
fangjia.mongodb.mongoUri=mongodb://${fangjia.mongo.host}:${fangjia.mongo.port},${fangjia.secondary.mongo.host}:${fangjia.secondary.mongo.port}/
fangjia.fs.mongo.host=fs.mongo.db.fangjia.com
fangjia.fs.mongo.port=27017
fangjia.fs.mongodb.mongoUri=mongodb://${fangjia.fs.mongo.host}:${fangjia.fs.mongo.port}/
fangjia.seabird.mongodb.mongoUri=mongodb://developer:G00IDCDEV12385@${fangjia.mongo.host}:${fangjia.mongo.port},${fangjia.secondary.mongo.host}:${fangjia.secondary.mongo.port}/
fangjia.knowbase.host=knowbase.db.fangjia.com
fangjia.knowbase.port=7180
fangjia.knowbase.url=${fangjia.knowbase.host}:${fangjia.knowbase.port}/
fangjia.environment=development
#fangjia.environment=production
@@ -18,8 +18,15 @@
</bean>
<import resource="classpath:spring-service.xml" />
<bean id="seabirdMongo" class="com.mongodb.MongoClient">
<constructor-arg ref="seabirdMongoUri" />
</bean>
<bean id="seabirdMongoUri" class="com.mongodb.MongoClientURI">
<constructor-arg type="java.lang.String" value="${fangjia.seabird.mongodb.mongoUri}"/>
</bean>
<bean id="seaweedDao" class="com.fangjia.dal.seaweed.dao.SeaweedDaoImpl">
<property name="mongo" ref="seabirdMongo" />
</bean>
<bean id="dataStore" class="com.fangjia.datastore.driver.DataStore">
<constructor-arg value="${fangjia.datastore.host}" />
<constructor-arg value="${fangjia.datastore.port}" />
@@ -29,20 +36,27 @@
<constructor-arg value="${fangjia.seaweed.host}" />
<constructor-arg value="${fangjia.seaweed.port}" />
</bean>
<bean id="mongo" class="com.mongodb.Mongo">
<constructor-arg index="0" value="${fangjia.mongo.host}" type="java.lang.String"/>
<constructor-arg index="1" value="${fangjia.mongo.port}" type="int"/>
<bean id="mongoUri" class="com.mongodb.MongoClientURI">
<constructor-arg type="java.lang.String" value="${fangjia.mongodb.mongoUri}"/>
</bean>
<bean id="mongo" class="com.mongodb.MongoClient">
<constructor-arg ref="mongoUri" />
</bean>
<bean id="logMongo" class="com.mongodb.Mongo">
<constructor-arg index="0" value="${fangjia.log.mongo.host}" type="java.lang.String"/>
<constructor-arg index="1" value="${fangjia.log.mongo.port}" type="int"/>
<bean id="logMongoUri" class="com.mongodb.MongoClientURI">
<constructor-arg type="java.lang.String" value="${fangjia.log.mongodb.mongoUri}"/>
</bean>
<bean id="fileMongo" class="com.mongodb.Mongo">
<constructor-arg index="0" value="${fangjia.file.mongo.host}" type="java.lang.String"/>
<constructor-arg index="1" value="${fangjia.file.mongo.port}" type="int"/>
<bean id="logMongo" class="com.mongodb.MongoClient">
<constructor-arg ref="logMongoUri" />
</bean>
<bean id="fileMongoUri" class="com.mongodb.MongoClientURI">
<constructor-arg type="java.lang.String" value="${fangjia.fs.mongodb.mongoUri}"/>
</bean>
<bean id="fileMongo" class="com.mongodb.MongoClient">
<constructor-arg ref="fileMongoUri" />
</bean>
<context:component-scan base-package="com.fangjia.dal.datastore.dao"
+30 -30
View File
@@ -25,12 +25,12 @@
</filter-class>
</filter>
<filter>
<filter-name>AccessLogFilter</filter-name>
<filter-class>
com.fangjia.geluke.filters.AccessLogFilter
</filter-class>
</filter>
<!-- <filter>-->
<!-- <filter-name>AccessLogFilter</filter-name>-->
<!-- <filter-class>-->
<!-- com.fangjia.geluke.filters.AccessLogFilter-->
<!-- </filter-class>-->
<!-- </filter>-->
<filter>
<filter-name>SpiderValidateFilter</filter-name>
@@ -47,20 +47,20 @@
</init-param>
</filter>
<filter>
<filter-name>AnalyticsFilter</filter-name>
<filter-class>
org.springframework.web.filter.DelegatingFilterProxy
</filter-class>
<init-param>
<param-name>targetBeanName</param-name>
<param-value>analyticsFilter</param-value>
</init-param>
<init-param>
<param-name>targetFilterLifecycle</param-name>
<param-value>true</param-value>
</init-param>
</filter>
<!-- <filter>-->
<!-- <filter-name>AnalyticsFilter</filter-name>-->
<!-- <filter-class>-->
<!-- org.springframework.web.filter.DelegatingFilterProxy-->
<!-- </filter-class>-->
<!-- <init-param>-->
<!-- <param-name>targetBeanName</param-name>-->
<!-- <param-value>analyticsFilter</param-value>-->
<!-- </init-param>-->
<!-- <init-param>-->
<!-- <param-name>targetFilterLifecycle</param-name>-->
<!-- <param-value>true</param-value>-->
<!-- </init-param>-->
<!-- </filter>-->
<filter>
<filter-name>sothis</filter-name>
@@ -80,21 +80,21 @@
<url-pattern>/*</url-pattern>
<dispatcher>REQUEST</dispatcher>
</filter-mapping>
<filter-mapping>
<filter-name>AccessLogFilter</filter-name>
<url-pattern>/*</url-pattern>
<dispatcher>REQUEST</dispatcher>
</filter-mapping>
<!-- <filter-mapping>-->
<!-- <filter-name>AccessLogFilter</filter-name>-->
<!-- <url-pattern>/*</url-pattern>-->
<!-- <dispatcher>REQUEST</dispatcher>-->
<!-- </filter-mapping>-->
<filter-mapping>
<filter-name>SpiderValidateFilter</filter-name>
<url-pattern>/*</url-pattern>
<dispatcher>REQUEST</dispatcher>
</filter-mapping>
<filter-mapping>
<filter-name>AnalyticsFilter</filter-name>
<url-pattern>/*</url-pattern>
<dispatcher>REQUEST</dispatcher>
</filter-mapping>
<!-- <filter-mapping>-->
<!-- <filter-name>AnalyticsFilter</filter-name>-->
<!-- <url-pattern>/*</url-pattern>-->
<!-- <dispatcher>REQUEST</dispatcher>-->
<!-- </filter-mapping>-->
<filter-mapping>
<filter-name>sothis</filter-name>
<url-pattern>/*</url-pattern>
@@ -23,58 +23,55 @@ id="changecity">
<li>
<div class="left">华东城市</div>
<div class="right">
<a name="shanghai" href="http://sh.${domain}${root}/">上海</a>
<a name="hangzhou" href="http://hz.${domain}${root}/">杭州</a>
<a name="nanjing" href="http://nj.${domain}${root}/">南京</a>
<a name="ningbo" href="http://nb.${domain}${root}/">宁波</a>
<a name="suzhou" href="http://suz.${domain}${root}/">苏州</a>
<a name="wuxi" href="http://wx.${domain}${root}/">无锡</a>
<a name="jinan" href="http://jn.${domain}${root}/">济南</a>
<a name="qingdao" href="http://qd.${domain}${root}/">青岛</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E4%B8%8A%E6%B5%B7">上海</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%9D%AD%E5%B7%9E">杭州</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%8D%97%E4%BA%AC">南京</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%AE%81%E6%B3%A2">宁波</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%97%A0%E9%94%A1">无锡</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%B5%8E%E5%8D%97">济南</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%9D%92%E5%B2%9B">青岛</a>
</div>
</li>
<li>
<div class="left">华北东北</div>
<div class="right">
<a name="beijing" href="http://bj.${domain}${root}/">北京</a>
<a name="tianjin" href="http://tj.${domain}${root}/">天津</a>
<a name="shenyang" href="http://sy.${domain}${root}/">沈阳</a>
<a name="shijiazhuang" href="http://sjz.${domain}${root}/">石家庄</a>
<a name="dalian" href="http://dl.${domain}${root}/">大连</a>
<a name="taiyuan" href="http://ty.${domain}${root}/">太原</a>
<a name="changchun" href="http://changchun.${domain}${root}/">长春</a>
<a name="haerbin" href="http://haerbin.${domain}${root}/">哈尔滨</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%8C%97%E4%BA%AC">北京</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%A4%A9%E6%B4%A5">天津</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%B2%88%E9%98%B3">沈阳</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E7%9F%B3%E5%AE%B6%E5%BA%84">石家庄</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%A4%A7%E8%BF%9E">大连</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%A4%AA%E5%8E%9F">太原</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%95%BF%E6%98%A5">长春</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%93%88%E5%B0%94%E6%BB%A8">哈尔滨</a>
</div>
</li>
<li>
<div class="left">华南华中</div>
<div class="right">
<a name="guangzhou" href="http://gz.${domain}${root}/">广州</a>
<a name="shenzhen" href="http://sz.${domain}${root}/">深圳</a>
<a name="wuhan" href="http://wh.${domain}${root}/">武汉</a>
<a name="sanya" href="http://sanya.${domain}${root}/">三亚</a>
<a name="fuzhou" href="http://fz.${domain}${root}/">福州</a>
<a name="changsha" href="http://cs.${domain}${root}/">长沙</a>
<a name="xiamen" href="http://xm.${domain}${root}/">厦门</a>
<a name="zhengzhou" href="http://zz.${domain}${root}/">郑州</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%B9%BF%E5%B7%9E">广州</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%B7%B1%E5%9C%B3">深圳</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%AD%A6%E6%B1%89">武汉</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E4%B8%89%E4%BA%9A">三亚</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E7%A6%8F%E5%B7%9E">福州</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%95%BF%E6%B2%99">长沙</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%8E%A6%E9%97%A8">厦门</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%83%91%E5%B7%9E">郑州</a>
</div>
</li>
<li>
<div class="left">西部城市</div>
<div class="right">
<a name="chengdu" href="http://cd.${domain}${root}/">成都</a>
<a name="xian" href="http://xa.${domain}${root}/">西安</a>
<a name="chongqing" href="http://cq.${domain}${root}/">重庆</a>
<a name="guiyang" href="http://guiyang.${domain}${root}/">贵阳</a>
<a name="kunming" href="http://kunming.${domain}${root}/">昆明</a>
<a name="yinchuan" href="http://yinchuan.${domain}${root}/">银川</a>
<a name="lanzhou" href="http://lanzhou.${domain}${root}/">兰州</a>
<a name="wulumuqi" href="http://xj.${domain}${root}/">乌鲁木齐</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%88%90%E9%83%BD">成都</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E8%A5%BF%E5%AE%89">西安</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%87%8D%E5%BA%86">重庆</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E8%B4%B5%E9%98%B3">贵阳</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%98%86%E6%98%8E">昆明</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%93%B6%E5%B7%9D">银川</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%85%B0%E5%B7%9E">兰州</a>
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E4%B9%8C%E9%B2%81%E6%9C%A8%E9%BD%90">乌鲁木齐</a>
</div>
</li>
<li><div class="left"></div><div class="right"><span class="more"><a href="http://www.${domain}${root}/">更多城市>></a></span></div></li>
</ul>
</div>
<#--地图城市切换 end-->
@@ -28,7 +28,7 @@ $(document).ready(function(){
$("#_h_logout").click(function(){
jQuery.ajax({
type: "get",
url: "http://" + location.host + "${root}/user/logout",
url: "http://geluke.fangjia.com/user/logout",
dataType:'json',
success: function(msg){
location.href = '${cityUrl}';
@@ -200,27 +200,9 @@
<script type="text/javascript">
$(".tab-pane a, .hotcity a").click(function(){
var cityName = this.innerHTML;
// var userName = '${loginUserName!}';
var name = this.name;
// if(cityName!='上海'&&userName=='xianlixiong'&&cityName!='北京'){
// cityName = '上海';
// name = 'sh';
// }
// if(cityName!='上海'&&userName=='pingping'){
// cityName = '上海';
// name = 'sh';
// }
// if((userName=='Angela'||userName=='jialun')&&cityName!='上海'&&cityName!='昆山'){
// cityName = '上海';
// name = 'sh';
// }
// $(".cityWidth").text(cityName);
var url = 'http://'+name+'.${domain}${root}/map/index?cityName='+encodeURIComponent(cityName)+"&t=" + new Date().getTime()
var url = 'http://geluke.fangjia.com/map/index?cityName='+encodeURIComponent(cityName)+"&t=" + new Date().getTime()
window.location.href = url;
});
</script>
</body>
File diff suppressed because it is too large Load Diff