Merge branch 'feature/monitor' into script_develop
# Conflicts: # projects/fangjia-script/pom.xml
This commit is contained in:
@@ -19,6 +19,7 @@
|
||||
<modules>
|
||||
<module>../../projects/fangjia-core</module>
|
||||
<module>../../projects/fangjia-dal</module>
|
||||
<module>../../projects/chafangtong-dal</module>
|
||||
<module>../../projects/fangjia-service</module>
|
||||
<module>../../projects/geluke-dal</module>
|
||||
<module>../../projects/geluke-service</module>
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: geluke
|
||||
namespace: fangjia
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: RollingUpdate
|
||||
rollingUpdate:
|
||||
maxSurge: 1
|
||||
maxUnavailable: 0
|
||||
revisionHistoryLimit: 5
|
||||
selector:
|
||||
matchLabels:
|
||||
project: geluke
|
||||
app: fangjia
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
project: geluke
|
||||
app: fangjia
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: nexus-registry-secret
|
||||
nodeSelector:
|
||||
app: fangjia
|
||||
containers:
|
||||
- name: geluke
|
||||
image: 192.168.0.19:9012/geluke:latest
|
||||
imagePullPolicy: Always
|
||||
resources:
|
||||
limits:
|
||||
memory: "3Gi"
|
||||
requests:
|
||||
memory: "2Gi"
|
||||
env:
|
||||
- name: JAVA_OPTS
|
||||
value: "-Xmx3072m -XX:MaxPermSize=256m
|
||||
-Dlog4j.configuration=file:/home/goojia/webapps/ROOT/WEB-INF/classes/log4j_linux.xml
|
||||
-Dfangjia.overridePropertiesLocation=/home/goojia/goojiaconfig/geluke.properties"
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 5
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: web
|
||||
protocol: TCP
|
||||
volumeMounts:
|
||||
- name: host-time
|
||||
mountPath: /etc/localtime
|
||||
- name: goojiaconfig
|
||||
mountPath: /home/goojia/goojiaconfig
|
||||
|
||||
volumes:
|
||||
- name: host-time
|
||||
hostPath:
|
||||
path: /etc/localtime
|
||||
- name: goojiaconfig
|
||||
cephfs:
|
||||
monitors:
|
||||
- 192.168.0.208:6789
|
||||
- 192.168.0.209:6789
|
||||
- 192.168.0.210:6789
|
||||
user: admin
|
||||
path: /k8s/goojiaconfig
|
||||
secretRef:
|
||||
name: ceph-secret
|
||||
readOnly: false
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: geluke
|
||||
namespace: fangjia
|
||||
labels:
|
||||
project: geluke
|
||||
app: fangjia
|
||||
spec:
|
||||
type: NodePort
|
||||
selector:
|
||||
project: geluke
|
||||
app: fangjia
|
||||
ports:
|
||||
- port: 8080
|
||||
targetPort: 8080
|
||||
nodePort: 30010
|
||||
@@ -1,63 +0,0 @@
|
||||
/**
|
||||
* Created by Administrator on 2017/5/9.
|
||||
*/
|
||||
import com.sun.org.apache.xerces.internal.impl.io.UTF8Reader;
|
||||
import com.sun.xml.internal.txw2.Document;
|
||||
import sun.nio.cs.ext.GB18030;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.InputStreamReader;
|
||||
import java.net.URL;
|
||||
import java.net.URLConnection;
|
||||
public class Test {
|
||||
String fixture_Date = null;
|
||||
String area = null;
|
||||
String unit_Price = null;
|
||||
String transaction_Price = null;
|
||||
static String sendGet(String url) {
|
||||
String result = "";
|
||||
// 定义一个缓冲字符输入流
|
||||
BufferedReader in = null;
|
||||
try {
|
||||
// 将string转成url对象
|
||||
URL realUrl = new URL(url);
|
||||
// 初始化一个链接到那个url的连接
|
||||
URLConnection connection = realUrl.openConnection();
|
||||
// 开始实际的连接
|
||||
connection.connect();
|
||||
// 初始化 BufferedReader输入流来读取URL的响应
|
||||
in = new BufferedReader(new InputStreamReader(connection.getInputStream(),"gbk"));
|
||||
// 用来临时存储抓取到的每一行的数据
|
||||
String line;
|
||||
while ((line = in.readLine()) != null)
|
||||
{ // 遍历抓取到的每一行并将其存储到result里面
|
||||
result += line;
|
||||
}
|
||||
Document document = Jsoup.parse(result, url);
|
||||
|
||||
parseListPage(document);
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
System.out.println("发送GET请求出现异常!" + e);
|
||||
e.printStackTrace(); }
|
||||
// 使用finally来关闭输入流
|
||||
finally {
|
||||
try {
|
||||
if (in != null) {
|
||||
in.close(); }
|
||||
} catch (Exception e2) {
|
||||
e2.printStackTrace();
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
public static void main(String[] args) {
|
||||
// 定义即将访问的链接
|
||||
String url = "http://dl.goufang.com/xq/1062/6";
|
||||
// 访问链接并获取页面内容
|
||||
String result = sendGet(url);
|
||||
System.out.println(result);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -122,10 +122,8 @@ public class District implements SeaweedEntity {
|
||||
private String houseStructure; //房屋结构
|
||||
private String source;
|
||||
private String propertyLife;
|
||||
//
|
||||
// //乡镇街道
|
||||
// private String township;
|
||||
//
|
||||
//乡镇街道
|
||||
private String township;
|
||||
|
||||
private String pinyin;
|
||||
|
||||
@@ -1873,14 +1871,14 @@ public class District implements SeaweedEntity {
|
||||
this.mappingAlias = mappingAlias;
|
||||
}
|
||||
|
||||
// @Column(name = "township")
|
||||
// public String getTownship() {
|
||||
// return township;
|
||||
// }
|
||||
//
|
||||
// public void setTownship(String township) {
|
||||
// this.township = township;
|
||||
// }
|
||||
@Column(name = "township")
|
||||
public String getTownship() {
|
||||
return township;
|
||||
}
|
||||
|
||||
public void setTownship(String township) {
|
||||
this.township = township;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -159,6 +159,11 @@
|
||||
<artifactId>jfreechart</artifactId>
|
||||
<version>1.5.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.xmlgraphics</groupId>
|
||||
<artifactId>batik-codec</artifactId>
|
||||
<version>1.11</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-core</artifactId>
|
||||
|
||||
+21
@@ -1705,6 +1705,25 @@ public class AddressController {
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object similarityAction(AddesssMatchModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
|
||||
mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空"));
|
||||
return mav;
|
||||
}
|
||||
model.setAddress(model.getAddress_1());
|
||||
String result1 = addressMatchProxyService.standard(convertModelToMap(model));
|
||||
model.setAddress(model.getAddress_2());
|
||||
String result2 = addressMatchProxyService.standard(convertModelToMap(model));
|
||||
if (result1 != null && result2 != null) {
|
||||
Map<String, Object> resultMap = addressService.addressSimilarity(result1, result2);
|
||||
mav.setModel(BasicModel.buildOKModel().setResult(resultMap));
|
||||
} else {
|
||||
mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中..."));
|
||||
}
|
||||
return mav;
|
||||
}
|
||||
|
||||
public Object parseCityAction(AddressModel model) {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
if (StringUtils.isBlank(model.getProvince())) {
|
||||
@@ -1750,6 +1769,7 @@ public class AddressController {
|
||||
|
||||
// matchParams.setUseMap(true);
|
||||
matchParams.setUseAliyunApi(true);
|
||||
// matchParams.setUseCrf(true);
|
||||
matchParams.setContainsLocation(model.isContainsLocation());
|
||||
Building building = addressParserService.standard(matchParams);
|
||||
Address address = building.getSearchAddress();
|
||||
@@ -1790,6 +1810,7 @@ public class AddressController {
|
||||
}
|
||||
}
|
||||
dto.append("buildingNumber", address.getBuildingNumber());
|
||||
dto.append("floor", address.getFloor());
|
||||
dto.append("unitNumber", address.getUnitNumber());
|
||||
dto.append("roomNumber", address.getRoomNumber());
|
||||
dto.append("formattedAddress", address.getFormattedAddress());
|
||||
|
||||
+2
-9
@@ -131,7 +131,7 @@ import com.google.common.collect.Sets;
|
||||
import com.mongodb.BasicDBObject;
|
||||
import com.mongodb.DBObject;
|
||||
|
||||
//@Authority(role = Role.USER)
|
||||
@Authority(role = Role.USER)
|
||||
public class PropertyController {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(PropertyController.class);
|
||||
@@ -4966,6 +4966,7 @@ public class PropertyController {
|
||||
* @throws NumberFormatException
|
||||
* @throws IllegalAccessException
|
||||
*/
|
||||
@Authority(role = Role.NONE)
|
||||
public Object rentReportAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
@@ -5164,7 +5165,6 @@ public class PropertyController {
|
||||
}
|
||||
|
||||
|
||||
|
||||
public Object evaluateRentAction(PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
RestfulModelAndView mav = new RestfulModelAndView();
|
||||
mav.setViewType("rest");
|
||||
@@ -5255,13 +5255,6 @@ public class PropertyController {
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public void setHouseLoudongService(HouseLoudongService houseLoudongService) {
|
||||
this.houseLoudongService = houseLoudongService;
|
||||
}
|
||||
|
||||
+66
-9
@@ -9,6 +9,7 @@ import com.fangjia.dal.mongo.model.Evaluation;
|
||||
import com.fangjia.dal.mongo.model.RentDealPrice;
|
||||
import com.fangjia.dal.seawater.model.Seawater;
|
||||
import com.fangjia.dal.seaweed.model.District;
|
||||
import com.fangjia.open.APIConstant;
|
||||
import com.fangjia.open.compatible.model.JsonModel;
|
||||
import com.fangjia.open.dto.DistrictHouseDtao;
|
||||
import com.fangjia.open.interceptors.AuthenticationBean;
|
||||
@@ -26,11 +27,17 @@ import com.fangjia.service.mongo.fangjia.FileService;
|
||||
import com.fangjia.service.mongo.fangjia.SeawaterService;
|
||||
import com.fangjia.service.util.DistrictScoreUtil;
|
||||
import com.lowagie.text.pdf.BaseFont;
|
||||
import org.apache.batik.transcoder.Transcoder;
|
||||
import org.apache.batik.transcoder.TranscoderException;
|
||||
import org.apache.batik.transcoder.TranscoderInput;
|
||||
import org.apache.batik.transcoder.TranscoderOutput;
|
||||
import org.apache.batik.transcoder.image.PNGTranscoder;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.lucene.index.Term;
|
||||
import org.apache.lucene.search.BooleanClause;
|
||||
import org.apache.lucene.search.BooleanQuery;
|
||||
import org.apache.lucene.search.TermQuery;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.sothis.core.util.CollectionUtils;
|
||||
@@ -41,9 +48,8 @@ import org.xhtmlrenderer.pdf.ITextRenderer;
|
||||
|
||||
import javax.servlet.http.HttpServletRequest;
|
||||
import javax.servlet.http.HttpServletResponse;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.io.*;
|
||||
import java.lang.reflect.InvocationTargetException;
|
||||
import java.text.ParseException;
|
||||
import java.util.*;
|
||||
|
||||
@@ -81,10 +87,29 @@ public class CcbRentController {
|
||||
return result;
|
||||
}
|
||||
|
||||
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
|
||||
public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
|
||||
String result = null;
|
||||
HttpRequest httpRequest = new HttpRequest();
|
||||
try {
|
||||
Map<String, String> headers = new HashMap<String, String>();
|
||||
// &__refresh_cache=1 ˢлº´æ
|
||||
String url = "http://open.fangjia.com/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN;
|
||||
Map<String, String> params = org.apache.commons.beanutils.BeanUtils.describe(model);
|
||||
result = httpRequest.post(url, params, headers,"UTF-8");
|
||||
} catch (NoSuchMethodException e) {
|
||||
e.printStackTrace();
|
||||
} catch (HttpRequestException e) {
|
||||
e.printStackTrace();
|
||||
} catch (InvocationTargetException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
|
||||
@Param(name = "name") String name, @Param(name = "address") String address,
|
||||
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
|
||||
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment) {
|
||||
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) {
|
||||
com.fangjia.open.model.PropertyEvaluationModel model = new com.fangjia.open.model.PropertyEvaluationModel();
|
||||
model.setCity(city);
|
||||
model.setDistrict(districtStr);
|
||||
@@ -261,7 +286,8 @@ public class CcbRentController {
|
||||
districtPriceMap.put("trends", trendsMap);
|
||||
|
||||
// 生成走势图图片,一遍转pdf
|
||||
String tradesImgUrl = buildTrendImage(trendsMap, room);
|
||||
//String tradesImgUrl = buildTrendImage(trendsMap, room);
|
||||
String tradesImgUrl = chartImgUrl;
|
||||
|
||||
//成交的租房房源
|
||||
List<RentDealPrice> rentTrades = rentDealPriceService.getRentDealPrices(district.getCity(), district.getRegion(), district.getName(), 5);
|
||||
@@ -426,23 +452,54 @@ public class CcbRentController {
|
||||
return evaluation;
|
||||
}
|
||||
|
||||
public Object saveHighChartAsImageAction(HttpServletRequest request, HttpServletResponse response) {
|
||||
String svg = request.getParameter("svg");
|
||||
ByteArrayOutputStream out = null;
|
||||
String url = null;
|
||||
try {
|
||||
svg = svg.replaceAll(":rect", "rect");
|
||||
Transcoder t = new PNGTranscoder();
|
||||
out = new ByteArrayOutputStream();
|
||||
TranscoderInput input = new TranscoderInput(new StringReader(svg));
|
||||
TranscoderOutput output = new TranscoderOutput(out);
|
||||
t.transcode(input, output);
|
||||
byte[] data = out.toByteArray();
|
||||
String id = new ObjectId().toString();
|
||||
fileService.save(id,"o", data, "image/png", "o");
|
||||
url = "http://img0.fangjia.com/stc/fs/" + DataUtil.encodePicUrl(id, "o", "house");
|
||||
} catch (TranscoderException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
try {
|
||||
out.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
JsonModel model = new JsonModel();
|
||||
AutoCompleteBo bo = new AutoCompleteBo();
|
||||
bo.setAddress(url);
|
||||
model.setJsonModel(bo);
|
||||
return model;
|
||||
}
|
||||
|
||||
public void downloadAction(HttpServletResponse response, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String district,
|
||||
@Param(name = "name") String name, @Param(name = "address") String address,
|
||||
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
|
||||
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment) throws IOException {
|
||||
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) throws IOException {
|
||||
response.setHeader("Content-disposition", "attachment; filename=report.pdf");
|
||||
response.setContentType("application/pdf;charset=UTF-8");
|
||||
response.setCharacterEncoding("UTF-8");
|
||||
OutputStream os = null;
|
||||
try {
|
||||
os = response.getOutputStream();
|
||||
ITextRenderer renderer = new ITextRenderer(35, 20);
|
||||
ITextRenderer renderer = new ITextRenderer(35, 24);
|
||||
ITextFontResolver fontResolver = renderer.getFontResolver();
|
||||
String fontUrl = ActionContext.getContext().getServletContext().getRealPath("/") + "fonts" + File.separator + "simsun.ttc";
|
||||
fontResolver.addFont(fontUrl, BaseFont.IDENTITY_H, BaseFont.NOT_EMBEDDED);
|
||||
final String path = "http://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath();
|
||||
String url = path + "/ccbRent/downloadPage?city=" + DataUtil.urlEncode(city) + "&district="+DataUtil.urlEncode(district)+"&name="+DataUtil.urlEncode(name);
|
||||
url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment;
|
||||
url += "&address=" + DataUtil.urlEncode(address) +"&size="+size+"&floor="+floor+"&height="+height+"&room="+room+"&hall="+hall+"&toilet="+toilet+"&direction="+direction+"&fitment="+fitment+"&chartImgUrl="+chartImgUrl;
|
||||
renderer.setDocument(url);
|
||||
renderer.layout();
|
||||
renderer.createPDF(os, true);
|
||||
|
||||
@@ -19,7 +19,24 @@ public class AddesssMatchModel extends FangjiaApiModel {
|
||||
private String province;
|
||||
private String debugName;
|
||||
private boolean useMap;
|
||||
private String address_1;
|
||||
private String address_2;
|
||||
|
||||
public String getAddress_1() {
|
||||
return address_1;
|
||||
}
|
||||
|
||||
public void setAddress_1(String address_1) {
|
||||
this.address_1 = address_1;
|
||||
}
|
||||
|
||||
public String getAddress_2() {
|
||||
return address_2;
|
||||
}
|
||||
|
||||
public void setAddress_2(String address_2) {
|
||||
this.address_2 = address_2;
|
||||
}
|
||||
public boolean isContainsLocation() {
|
||||
return containsLocation;
|
||||
}
|
||||
|
||||
@@ -2,23 +2,24 @@ package com.fangjia.open.util;
|
||||
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.service.mongo.fangjia.FileService;
|
||||
import org.apache.batik.transcoder.Transcoder;
|
||||
import org.apache.batik.transcoder.TranscoderException;
|
||||
import org.apache.batik.transcoder.TranscoderInput;
|
||||
import org.apache.batik.transcoder.TranscoderOutput;
|
||||
import org.apache.batik.transcoder.image.JPEGTranscoder;
|
||||
import org.apache.batik.transcoder.image.PNGTranscoder;
|
||||
import org.bson.types.ObjectId;
|
||||
import org.jfree.chart.ChartFactory;
|
||||
import org.jfree.chart.ChartPanel;
|
||||
import org.jfree.chart.ChartUtilities;
|
||||
import org.jfree.chart.JFreeChart;
|
||||
import org.jfree.chart.axis.NumberAxis;
|
||||
import org.jfree.chart.plot.CategoryPlot;
|
||||
import org.jfree.chart.plot.PlotOrientation;
|
||||
import org.jfree.chart.renderer.category.LineAndShapeRenderer;
|
||||
import org.jfree.chart.ui.ApplicationFrame;
|
||||
import org.jfree.data.category.DefaultCategoryDataset;
|
||||
import org.jfree.ui.RefineryUtilities;
|
||||
|
||||
import java.awt.*;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.io.*;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@@ -99,6 +100,42 @@ public class LineChartsUtils {
|
||||
return null;
|
||||
}
|
||||
|
||||
public static void svg2ImgTest() {
|
||||
String svg = "<svg height=\"200\" xmlns=\"http://www.w3.org/2000/svg\"><polygon points=\"10,10 75,150 150,60\" style=\"fill:#63BCF7;stroke:black;stroke-width:3\"/></svg>";
|
||||
String type="image/png";
|
||||
String filename="svg";
|
||||
if (null != type && null != svg) {
|
||||
svg = svg.replaceAll(":rect", "rect");
|
||||
String ext = "";
|
||||
Transcoder t = null;
|
||||
if (type.equals("image/png")) {
|
||||
ext = "png";
|
||||
t = new PNGTranscoder();
|
||||
}
|
||||
else if (type.equals("image/jpeg")) {
|
||||
ext = "jpg";
|
||||
t = new JPEGTranscoder();
|
||||
}
|
||||
if (null != t) {
|
||||
OutputStream out = null;
|
||||
try {
|
||||
out = new FileOutputStream("D:\\"+filename+"."+ext);
|
||||
} catch (FileNotFoundException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
TranscoderInput input = new TranscoderInput(new StringReader(svg));
|
||||
TranscoderOutput output = new TranscoderOutput(out);
|
||||
try{
|
||||
t.transcode(input, output);
|
||||
System.out.println("out");
|
||||
}
|
||||
catch (TranscoderException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 定义图表对象
|
||||
* @param rowKey 曲线名,如一居室
|
||||
|
||||
@@ -18,22 +18,28 @@
|
||||
body {
|
||||
overflow: hidden;
|
||||
overflow-y: scroll;
|
||||
background: #f8f8f8;
|
||||
background: white;
|
||||
}
|
||||
.main {
|
||||
margin: 4px 8px 4px 8px;
|
||||
background: #f8f8f8;
|
||||
background: white;
|
||||
}
|
||||
.title {
|
||||
position: relative;
|
||||
color: #1E90FF;
|
||||
font-size: 20px;
|
||||
font-weight: bold;
|
||||
}
|
||||
.title1 {
|
||||
font-size: 14px;
|
||||
font-size: 18px;
|
||||
font-weight: bold;
|
||||
color: #1E90FF;
|
||||
margin-top: 2px;
|
||||
margin-bottom: 2px;
|
||||
}
|
||||
.p {
|
||||
font-size: 12px;
|
||||
font-size: 18px;
|
||||
line-height: 30px;
|
||||
}
|
||||
.table {
|
||||
width: 100%;
|
||||
@@ -77,11 +83,11 @@
|
||||
<div class="title1">2 评估时间</div>
|
||||
<div class="p">
|
||||
<span name="cal_date"></span><br/>
|
||||
评估结果<br/>
|
||||
楼盘名称:<span name="name"></span><br/>
|
||||
楼盘地址:<span name="address"></span><br/>
|
||||
租金估值(RMB):<span name="totalPrice"></span>元/月<br/>
|
||||
大写(人民币):<span name="totalPriceUpper"></span><br/>
|
||||
<b>评估结果</b><br/>
|
||||
<b>楼盘名称</b>:<span name="name"></span><br/>
|
||||
<b>楼盘地址</b>:<span name="address"></span><br/>
|
||||
<b>租金估值(RMB)</b>:<span name="totalPrice"></span>元/月<br/>
|
||||
<b>大写(人民币)</b>:<span name="totalPriceUpper"></span><br/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
@@ -91,20 +97,21 @@
|
||||
<div>
|
||||
<div class="title1">4 房产状况</div>
|
||||
<div class="p">
|
||||
详细地址:<span name="address"></span><br/>
|
||||
楼盘名称:<span name="name"></span><br/>
|
||||
室号:<span name="room"></span><br/>
|
||||
朝向:<span name="direction"></span><br/>
|
||||
建筑面积:<span name="size"></span>平米<br/>
|
||||
所在楼层:<span name="floor"></span> 层 <span name="height"></span>总层数<br/>
|
||||
电梯配套:<span name="hasLift"></span><br/>
|
||||
装修情况:<span name="fitment"></span><br/>
|
||||
建筑年代:<span name="buildYear"></span><br/>
|
||||
<b>详细地址</b>:<span name="address"></span><br/>
|
||||
<b>楼盘名称</b>:<span name="name"></span><br/>
|
||||
<b>室号</b>:<span name="room"></span><br/>
|
||||
<b>朝向</b>:<span name="direction"></span><br/>
|
||||
<b>建筑面积</b>:<span name="size"></span>平米<br/>
|
||||
<b>所在楼层</b>:<span name="floor"></span> 层 <span name="height"></span>总层数<br/>
|
||||
<b>电梯配套</b>:<span name="hasLift"></span><br/>
|
||||
<b>装修情况</b>:<span name="fitment"></span><br/>
|
||||
<b>建筑年代</b>:<span name="buildYear"></span><br/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">5 历史租金</div>
|
||||
<div class="charts" id="districtRentTrendsChart"></div>
|
||||
<input type="hidden" id="chartImgUrl"/>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">6 已租房源</div>
|
||||
@@ -179,7 +186,8 @@
|
||||
<script type='text/javascript' src='http://g.alicdn.com/sj/lib/zepto/zepto.js' charset='utf-8'></script>
|
||||
<script type='text/javascript' src='http://g.alicdn.com/msui/sm/0.6.2/js/sm.js' charset='utf-8'></script>
|
||||
<!-- 暂时注释 -->
|
||||
<script type="text/javascript" src="http://cdn.hcharts.cn/highcharts/highcharts.js"></script>
|
||||
<script type="text/javascript" src="http://cdn.highcharts.com.cn/highcharts/9.0.1/highcharts.js"></script>
|
||||
<script type="text/javascript" src="https://code.highcharts.com.cn/highcharts/modules/exporting.js"></script>
|
||||
<script type='text/javascript' src='../static/js/wx.js' charset='utf-8'></script>
|
||||
<script type="text/javascript" src="../static/js/trend-charts.js"></script>
|
||||
<script src="https://html2canvas.hertzen.com/dist/html2canvas.js"></script>
|
||||
@@ -218,9 +226,11 @@
|
||||
// jspdf.addFont('SourceHanSans-Normal.ttf', 'SourceHanSans-Normal', 'normal');
|
||||
// jspdf.setFont('SourceHanSans-Light');
|
||||
//console.log(jspdf.getFontList());
|
||||
var chart;
|
||||
$("#download").bind('click', function(){
|
||||
var chartImgUrl = $('#chartImgUrl').val();
|
||||
window.location.href = "download?city="+params.city+"&district="+params.district+"&name="+params.name+"&address="+params.address+"&size="+params.size+"&floor="+params.floor
|
||||
+"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment;
|
||||
+"&height="+params.height+"&room="+params.room+"&hall="+params.hall+"&toilet="+params.toilet+"&direction="+params.direction+"&fitment="+params.fitment+"&chartImgUrl=" + chartImgUrl;
|
||||
// jspdf.html(document.body, {
|
||||
// callback: function (doc) {
|
||||
// doc.save("租金报告.pdf");
|
||||
@@ -319,9 +329,9 @@
|
||||
<td class="td01">' + d.city + '</td> \
|
||||
<td class="td02">' + d.region + '</td> \
|
||||
<td>' + d.districtName + '</td> \
|
||||
<td>' + d.totalPrice + '</td> \
|
||||
<td>' + d.area + '</td> \
|
||||
<td>' + d.room + '</td> \
|
||||
<td>' + (d.totalPrice != undefined ? d.totalPrice : '') + '</td> \
|
||||
<td>' + (d.area != undefined ? d.area : '') + '</td> \
|
||||
<td>' + (d.room != undefined ? d.room : '') + '</td> \
|
||||
<td>' + hall + '</td> \
|
||||
<td>' + fitment + '</td> \
|
||||
<td>' + formatDate(d.tradeDate) + '</td> \
|
||||
@@ -340,10 +350,10 @@
|
||||
<td class="td01">' + d.city + '</td> \
|
||||
<td class="td02">' + d.region + '</td> \
|
||||
<td>' + d.districtName + '</td> \
|
||||
<td>' + d.totalPrice + '</td> \
|
||||
<td>' + d.area + '</td> \
|
||||
<td>' + d.room + '</td> \
|
||||
<td>' + d.hall + '</td> \
|
||||
<td>' + (d.totalPrice != undefined ? d.totalPrice : '') + '</td> \
|
||||
<td>' + (d.area != undefined ? d.area : '') + '</td> \
|
||||
<td>' + (d.room != undefined ? d.room : '') + '</td> \
|
||||
<td>' + (d.hall != undefined ? d.hall : '')+ '</td> \
|
||||
<td>' + fitment + '</td> \
|
||||
<td>' + date + '</td> \
|
||||
</tr>');
|
||||
@@ -359,9 +369,9 @@
|
||||
$('#sameDistricts').append('<tr class=""> \
|
||||
<td class="td01">' + d.city + '</td> \
|
||||
<td class="td02">' + d.region + '</td> \
|
||||
<td>' + d.districtName + '</td> \
|
||||
<td>' + d.roomPrice + '</td> \
|
||||
<td>' + d.room + '</td> \
|
||||
<td>' + (d.districtName != undefined ? d.districtName:'' ) + '</td> \
|
||||
<td>' + (d.roomPrice != undefined ? d.roomPrice:'') + '</td> \
|
||||
<td>' + (d.room != undefined ? d.room:'') + '</td> \
|
||||
<td>1</td> \
|
||||
</tr>');
|
||||
})
|
||||
@@ -426,8 +436,9 @@
|
||||
var config = {yaxis : yaxis};
|
||||
createHighChartNow(renderId, data, config);
|
||||
};
|
||||
|
||||
function createHighChartNow(renderId, data, config) {
|
||||
var chart = new Highcharts.Chart({
|
||||
chart = new Highcharts.Chart({
|
||||
credits:{
|
||||
enabled:true,
|
||||
text: '',
|
||||
@@ -446,6 +457,9 @@
|
||||
renderTo: renderId,
|
||||
type: 'spline'
|
||||
},
|
||||
exporting: {
|
||||
enabled: false
|
||||
},
|
||||
title: {
|
||||
text: data.title,
|
||||
floating: false,
|
||||
@@ -479,7 +493,26 @@
|
||||
}
|
||||
}
|
||||
});
|
||||
saveHighChartImg();
|
||||
};
|
||||
|
||||
function saveHighChartImg() {
|
||||
var svg = chart.getSVG();
|
||||
$.ajax({
|
||||
// 注意需要加..
|
||||
url: "saveHighChartAsImage",
|
||||
data: {
|
||||
svg:svg
|
||||
},
|
||||
dataType: "json",
|
||||
type: "post",
|
||||
success: function (ret) {
|
||||
if (ret.address != null) {
|
||||
$('#chartImgUrl').val(ret.address)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
$(function () {
|
||||
init();
|
||||
});
|
||||
|
||||
@@ -14,221 +14,191 @@
|
||||
|
||||
<link rel="stylesheet" href="http://g.alicdn.com/msui/sm/0.6.2/css/sm.css"/>
|
||||
<link rel="stylesheet" href="../static/style/wx.css"/>
|
||||
<link rel="stylesheet" href="../static/style/report.css"/>
|
||||
<style type="text/css">
|
||||
body {
|
||||
overflow: hidden;
|
||||
overflow-y: scroll;
|
||||
background: #f8f8f8;
|
||||
/** 特别注意,否则中文不生效 **/
|
||||
font-family: SimSun;
|
||||
}
|
||||
.main {
|
||||
margin: 4px 8px 4px 8px;
|
||||
background: #f8f8f8;
|
||||
}
|
||||
.title {
|
||||
position: relative;
|
||||
color: #1E90FF;
|
||||
}
|
||||
.title1 {
|
||||
font-size: 14px;
|
||||
color: #1E90FF;
|
||||
}
|
||||
.p {
|
||||
font-size: 12px;
|
||||
}
|
||||
.table {
|
||||
width: 100%;
|
||||
overflow-y: hidden;
|
||||
overflow-x: scroll
|
||||
}
|
||||
.inline {
|
||||
width: 550px;
|
||||
border-spacing: 0;
|
||||
text-align: left;
|
||||
}
|
||||
.table .inline th {
|
||||
color: #999;
|
||||
font-weight: normal;
|
||||
line-height: 30px;
|
||||
text-align: left;
|
||||
}
|
||||
.download {
|
||||
float: right;
|
||||
color: #3d4145;
|
||||
border: solid 1px #333;
|
||||
order-radius: 3px;
|
||||
font-size: 12px;
|
||||
line-height: 1rem;
|
||||
width: 43px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="main">
|
||||
<div class="title">
|
||||
估值报告
|
||||
<div class="content">
|
||||
<div class="t1">
|
||||
<em>1</em>估价结果
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">1 评估申明</div>
|
||||
<div class="p">建设银行遵循中立、客观、公正的原则,通过周密细致的测算,评估求得分析对象的房地产市场参考。</div>
|
||||
<div class="st1"><em>1.1</em>评估申明:</div>
|
||||
<div class="text1">建设银行遵循中立、客观、公正的原则,通过周密细致的测算,评估求得分析对象的房<br/>地产市场参考。</div>
|
||||
|
||||
<div class="st1"><em>1.2</em>评估时间:</div>
|
||||
<div class="text1"><#if result.cal_date??>${result.cal_date}</#if></div>
|
||||
<div class="st1"><em>1.3</em>评估结果:</div>
|
||||
<div class="text2">
|
||||
<b>楼盘名称</b>:<#if result.name??>${result.name}</#if><br/><br/>
|
||||
<b>楼盘地址</b>:<#if result.address??>${result.address}</#if><br/><br/>
|
||||
<b>租金估值(RMB)</b>:<#if result.totalPrice??>${result.totalPrice}</#if>元/月<br/><br/>
|
||||
<b>大写(人民币)</b>:<#if result.totalPriceUpper??>${result.totalPriceUpper}</#if><br/><br/>
|
||||
</div>
|
||||
<div class="st1"><em>1.4</em>报告有效期:</div>
|
||||
<div class="text1" style="margin-bottom: 30px">本报告在市场无较大波动的情况下,该评估价格在报告出具起三个月内有效。</div>
|
||||
<div class="t1"><em>2</em>房产状况</div>
|
||||
<div class="c1">
|
||||
<table class="distTab" width="700" border="0" cellspacing="0" cellpadding="0">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="textTd" style="width: 15%;">楼盘名称</td>
|
||||
<td class="textRight1" style="width: 35%;"><#if result.name??>${result.name}</#if></td>
|
||||
<td class="textTd" style="width: 15%;">地址</td>
|
||||
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.address??>${result.address}</#if></td>
|
||||
</tr>
|
||||
<tr class="tdbg">
|
||||
<td class="textTd">室号</td>
|
||||
<td class="textRight1"><#if result.room??>${result.room}</#if></td>
|
||||
<td class="textTd">朝向</td>
|
||||
<td class="textRight2" colspan="3"><#if result.direction??>${result.direction}</#if></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="textTd" style="width: 15%;">建筑面积</td>
|
||||
<td class="textRight1" style="width: 35%;"><#if result.size??>${result.size}</#if>平米</td>
|
||||
<td class="textTd" style="width: 15%;">所在楼层</td>
|
||||
<td class="textRight2" colspan="3" style="width: 35%;"><#if result.floor??>${result.floor}</#if> 层 <#if result.height??>${result.height}</#if>总层数</td>
|
||||
</tr>
|
||||
<tr class="tdbg">
|
||||
<td class="textTd">电梯配套</td>
|
||||
<td class="textRight1"><#if result.hasLift??>${result.hasLift}</#if></td>
|
||||
<td class="textTd">装修情况</td>
|
||||
<td class="textRight2" colspan="3"><#if result.fitment??>${result.fitment}</#if></td>
|
||||
</tr>
|
||||
<tr class="tdbg">
|
||||
<td class="textTd">建筑年代</td>
|
||||
<td class="textRight1"><#if result.buildYear??>${result.buildYear}</#if></td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="t1">
|
||||
<em>3</em>历史租金
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">2 评估时间</div>
|
||||
<div class="p">
|
||||
<span name="cal_date"><#if result.cal_date??>${result.cal_date}</#if></span><br/>
|
||||
评估结果<br/>
|
||||
楼盘名称:<span name="name"><#if result.name??>${result.name}</#if></span><br/>
|
||||
楼盘地址:<span name="address"><#if result.address??>${result.address}</#if></span><br/>
|
||||
租金估值(RMB):<span name="totalPrice"><#if result.totalPrice??>${result.totalPrice}</#if></span>元/月<br/>
|
||||
大写(人民币):<span name="totalPriceUpper"><#if result.totalPriceUpper??>${result.totalPriceUpper}</#if></span><br/>
|
||||
</div>
|
||||
<#if result.tradesImgUrl??>
|
||||
<#-- <img width="800px" height="300px" src="../static/images/test.png"></img>-->
|
||||
<img width="600px" height="300px" src="${result.tradesImgUrl}"></img>
|
||||
<#else>
|
||||
<div class="p table">
|
||||
<table class="inline">
|
||||
<tbody><tr><td>暂无数据</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</#if>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">3 报告有效期</div>
|
||||
<div class="p">本报告在市场无较大波动的情况下,该评估价格在报告出具起三个月内有效。</div>
|
||||
<div class="t1">
|
||||
<em>4</em>已租房源
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">4 房产状况</div>
|
||||
<div class="p">
|
||||
详细地址:<span name="address"><#if result.address??>${result.address}</#if></span><br/>
|
||||
楼盘名称:<span name="name"><#if result.name??>${result.name}</#if></span><br/>
|
||||
室号:<span name="room"><#if result.room??>${result.room}</#if></span><br/>
|
||||
朝向:<span name="direction"><#if result.direction??>${result.direction}</#if></span><br/>
|
||||
建筑面积:<span name="size"><#if result.size??>${result.size}</#if></span>平米<br/>
|
||||
所在楼层:<span name="floor"><#if result.floor??>${result.floor}</#if></span> 层 <span name="height"><#if result.height??>${result.height}</#if></span>总层数<br/>
|
||||
电梯配套:<span name="hasLift"><#if result.hasLift??>${result.hasLift}</#if></span><br/>
|
||||
装修情况:<span name="fitment"><#if result.fitment??>${result.fitment}</#if></span><br/>
|
||||
建筑年代:<span name="buildYear"><#if result.buildYear??>${result.buildYear}</#if></span><br/>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">5 历史租金</div>
|
||||
<div>
|
||||
<#if result.tradesImgUrl??>
|
||||
<#-- <img width="800px" height="300px" src="../static/images/test.png"></img>-->
|
||||
<img width="800px" height="300px" src="${result.tradesImgUrl}"></img>
|
||||
<#else>
|
||||
<div class="p table">
|
||||
<table class="inline">
|
||||
<tbody><tr><td>暂无数据</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</#if>
|
||||
</div>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">6 已租房源</div>
|
||||
<div class="p table">
|
||||
<table class="inline">
|
||||
<thead>
|
||||
<tr class="bg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>成交价</th>
|
||||
<th>面积</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
<th>装修</th>
|
||||
<th>成交日期</th>
|
||||
<div style="margin-bottom: 30px;">
|
||||
<table class="distTab" width="780" border="0" cellspacing="0" cellpadding="0">
|
||||
<thead>
|
||||
<tr class="tdbg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>成交价</th>
|
||||
<th>面积</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
<th>装修</th>
|
||||
<th>成交日期</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<#if (result.trades??&&result.trades?size>0)>
|
||||
<#list result.trades as bo>
|
||||
<tr class="tdbg">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.area??>${bo.area}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}</#if></td>
|
||||
<td><#if bo.fitment??>${bo.fitment}</#if></td>
|
||||
<td><#if bo.source??>${bo.source}</#if></td>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<#if (result.trades??&&result.trades?size>0)>
|
||||
<#list result.trades as bo>
|
||||
<tr class="">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.area??>${bo.area}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}</#if></td>
|
||||
<td><#if bo.fitment??>${bo.fitment}</#if></td>
|
||||
<td><#if bo.source??>${bo.source}</#if></td>
|
||||
</tr>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">7 样本参考</div>
|
||||
<div class="p table">
|
||||
<table class="inline">
|
||||
<thead>
|
||||
<tr class="bg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>租金</th>
|
||||
<th>面积</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
<th>装修</th>
|
||||
<th>成交日期</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="referenceHouses">
|
||||
<#if (result.houses??&&result.houses?size>0)>
|
||||
<#list result.houses as bo>
|
||||
<tr class="">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.area??>${bo.area}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}</#if></td>
|
||||
<td><#if bo.fitment??>${bo.fitment}</#if></td>
|
||||
<td><#if bo.root??>${bo.root}</#if></td>
|
||||
</tr>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="t1">
|
||||
<em>5</em>样本参考
|
||||
</div>
|
||||
<div>
|
||||
<div class="title1">8 周边同类小区租金</div>
|
||||
<div class="p table">
|
||||
<table class="inline" style="width: 300px">
|
||||
<thead>
|
||||
<tr class="bg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>平米租金</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="sameDistricts">
|
||||
<#if (result.districts??&&result.districts?size>0)>
|
||||
<#list result.districts as bo>
|
||||
<tr class="">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}</#if></td>
|
||||
</tr>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div style="margin-bottom: 30px;">
|
||||
<table class="distTab" width="780" border="0" cellspacing="0" cellpadding="0">
|
||||
<thead>
|
||||
<tr class="tdbg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>租 金</th>
|
||||
<th>面积</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
<th>装修</th>
|
||||
<th>成交日期</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="referenceHouses">
|
||||
<#if (result.houses??&&result.houses?size>0)>
|
||||
<#list result.houses as bo>
|
||||
<tr class="tdbg">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.area??>${bo.area}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}</#if></td>
|
||||
<td><#if bo.fitment??>${bo.fitment}</#if></td>
|
||||
<td><#if bo.root??>${bo.root}</#if></td>
|
||||
</tr>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="t1">
|
||||
<em>6</em>周边同类小区租金
|
||||
</div>
|
||||
<div style="margin-bottom: 30px;">
|
||||
<table class="distTab" width="500" border="0" cellspacing="0" cellpadding="0">
|
||||
<thead>
|
||||
<tr class="tdbg">
|
||||
<th class="th01">城市</th>
|
||||
<th>区域</th>
|
||||
<th>小区名</th>
|
||||
<th>平米租金</th>
|
||||
<th>室</th>
|
||||
<th>厅</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="sameDistricts">
|
||||
<#if (result.districts??&&result.districts?size>0)>
|
||||
<#list result.districts as bo>
|
||||
<tr class="tdbg">
|
||||
<td class="td01"><#if bo.city??>${bo.city}</#if></td>
|
||||
<td class="td02"><#if bo.region??>${bo.region}</#if></td>
|
||||
<td><#if bo.districtName??>${bo.districtName}</#if></td>
|
||||
<td><#if bo.totalPrice??>${bo.totalPrice}</#if></td>
|
||||
<td><#if bo.room??>${bo.room}</#if></td>
|
||||
<td><#if bo.hall??>${bo.hall}<#else>1</#if></td>
|
||||
</tr>
|
||||
</#list>
|
||||
<#else>
|
||||
<tr><td>暂无数据</td></tr>
|
||||
</#if>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
|
||||
@@ -260,7 +260,7 @@
|
||||
$("#evaluateBtn").prop("disabled", true)
|
||||
$.ajax({
|
||||
// 注意需要加..
|
||||
url: '../property/evaluateRent',
|
||||
url: '../ccbRent/evaluateRent',
|
||||
data:$("#evaluateForm").serialize(),
|
||||
dataType:"json",
|
||||
type:"post",
|
||||
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 3.1 KiB |
@@ -0,0 +1,141 @@
|
||||
body {
|
||||
overflow: hidden;
|
||||
overflow-y: scroll;
|
||||
background: white;
|
||||
/** 特别注意,否则中文不生效 **/
|
||||
font-family: SimSun;
|
||||
}
|
||||
.content {
|
||||
width: 840px;
|
||||
height: 100%;
|
||||
margin: 0 auto;
|
||||
background: #fff;
|
||||
padding-left: 20px;
|
||||
}
|
||||
.content .t1 {
|
||||
background: url(../images/fjt/bg.png) 0 0 no-repeat;
|
||||
width: 820px;
|
||||
height: 42px;
|
||||
color: #704f1c;
|
||||
font-size: 20px;
|
||||
font-weight: bold;
|
||||
text-indent: 14px;
|
||||
line-height: 46px;
|
||||
margin-bottom: 25px;
|
||||
}
|
||||
.content .t1 em {
|
||||
font-family: "Arial";
|
||||
color: #FFF;
|
||||
margin-right: 20px;
|
||||
}
|
||||
em {
|
||||
font-style: normal;
|
||||
}
|
||||
.content .st1 {
|
||||
background: url(../images/fjt/bg.png) 0 -60px no-repeat;
|
||||
width: 172px;
|
||||
height: 40px;
|
||||
color: #704f1c;
|
||||
font-size: 18px;
|
||||
font-weight: bold;
|
||||
text-indent: 7px;
|
||||
line-height: 46px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
.content .st1 em {
|
||||
font-family: "Arial";
|
||||
color: #FFF;
|
||||
margin-right: 14px;
|
||||
}
|
||||
.content .text1 {
|
||||
font-size: 18px;
|
||||
line-height: 26px;
|
||||
margin: 0 20px 10px 50px;
|
||||
word-wrap: break-word;
|
||||
}
|
||||
.text1 td {
|
||||
text-align: center
|
||||
}
|
||||
.content .text2 {
|
||||
font-size: 18px;
|
||||
line-height: 40px;
|
||||
padding: 10px 20px 10px 50px;
|
||||
background: #eee;
|
||||
width: 750px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
.content .text3 {
|
||||
font-size: 16px;
|
||||
line-height: 40px;
|
||||
padding: 10px 20px 10px 50px;
|
||||
background: #eee;
|
||||
width: 750px;
|
||||
margin-bottom: 30px;
|
||||
}
|
||||
.content .c1 {
|
||||
height: 200px;
|
||||
}
|
||||
.content .distTab {
|
||||
font-size: 18px
|
||||
}
|
||||
.content .distTab .textTd {
|
||||
font-weight: bold;
|
||||
}
|
||||
.content .distTab tr td {
|
||||
height: 30px;
|
||||
}
|
||||
.content .distTab tr th {
|
||||
text-align: left;
|
||||
}
|
||||
.content .distTab .textRight1 {
|
||||
text-align: right;
|
||||
padding-right: 80px;
|
||||
}
|
||||
.content .distTab .textRight2 {
|
||||
text-align: right;
|
||||
padding-right: 5px;
|
||||
}
|
||||
.title {
|
||||
position: relative;
|
||||
color: #1E90FF;
|
||||
}
|
||||
.title1 {
|
||||
font-size: 19px;
|
||||
color: #1E90FF;
|
||||
margin-top: 3px;
|
||||
margin-bottom: 3px;
|
||||
}
|
||||
.p {
|
||||
font-size: 16px;
|
||||
}
|
||||
table {
|
||||
border-collapse: separate;
|
||||
text-indent: initial;
|
||||
border-spacing: 2px;
|
||||
}
|
||||
tbody {
|
||||
display: table-row-group;
|
||||
vertical-align: middle;
|
||||
border-color: inherit;
|
||||
}
|
||||
.inline {
|
||||
width: 550px;
|
||||
border-spacing: 0;
|
||||
text-align: left;
|
||||
}
|
||||
.table .inline th {
|
||||
color: #999;
|
||||
font-weight: normal;
|
||||
line-height: 30px;
|
||||
text-align: left;
|
||||
}
|
||||
.download {
|
||||
float: right;
|
||||
color: #3d4145;
|
||||
border: solid 1px #333;
|
||||
order-radius: 3px;
|
||||
font-size: 12px;
|
||||
line-height: 1rem;
|
||||
width: 43px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
@@ -32,6 +32,7 @@ public enum Watcher {
|
||||
BaYi('15300424133','bayi@fangjia.com'),
|
||||
WeiLei('18101751850','weilei@fangjia.com'),
|
||||
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
|
||||
ShenJian('13764369683', 'shenjian@fangjia.com')
|
||||
private final String mobile;
|
||||
private final String email;
|
||||
|
||||
|
||||
+16
-1
@@ -218,7 +218,7 @@ class CommonDatautils {
|
||||
q.put("visible", 0);
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "工商大楼");
|
||||
// q.put("name","¶«Ô¢Ð´å")
|
||||
|
||||
DBObject keys = new BasicDBObject();
|
||||
if(fields){
|
||||
@@ -1392,6 +1392,21 @@ class CommonDatautils {
|
||||
}
|
||||
|
||||
|
||||
public static def queryDistrictPriceByWeekend(city, region, name, weekend, limit){
|
||||
DBObject queryCondition = new BasicDBObject()
|
||||
queryCondition.put("city", city);
|
||||
queryCondition.put("region", region);
|
||||
queryCondition.put("district_name", name);
|
||||
if(weekend){
|
||||
queryCondition.put('weekend', new BasicDBObject('$gte', weekend));
|
||||
}
|
||||
DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", 1)).limit(limit)
|
||||
def dbo = null;
|
||||
while(cur.hasNext()) {
|
||||
dbo = cur.next()
|
||||
}
|
||||
return dbo;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package com.fangjia.script.companyCrawl
|
||||
|
||||
import net.sf.json.JSONObject
|
||||
import org.jsoup.Jsoup
|
||||
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AiqichaCrawl {
|
||||
|
||||
def static final proxyPort = 9020;
|
||||
def static final proxyServer = "http-dyn.abuyun.com";
|
||||
def static final proxyUser = "HZU1XU28H3G4Z44D";
|
||||
def static final proxyPass = "466C86ED77FE43D4";
|
||||
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
|
||||
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
|
||||
public final static int AbuyunMaxCount = 10;
|
||||
|
||||
static main(args) {
|
||||
def task = new AiqichaCrawl();
|
||||
task.process()
|
||||
}
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
def url = "https://aiqicha.baidu.com/s/l?q=%E6%96%B0%E5%8F%8B&t=&p=3&s=10&o=0&f=%7B%7D "
|
||||
for(def key : companyKeys){
|
||||
def queryBody = null
|
||||
def s = crawl(url, queryBody);
|
||||
if(s){
|
||||
print s
|
||||
|
||||
|
||||
}
|
||||
println s
|
||||
}
|
||||
}
|
||||
|
||||
def crawl(url, queryBody){
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","aiqicha.baidu.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
headers.put("sec-ch-ua",'"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"')
|
||||
headers.put("sec-ch-ua-mobile","?0")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
headers.put("Sec-Fetch-Site","same-origin")
|
||||
headers.put("Sec-Fetch-Mode","navigate")
|
||||
headers.put("Sec-Fetch-User","?1")
|
||||
headers.put("Sec-Fetch-Dest","document")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
|
||||
String s = doPost(url, queryBody, headers)
|
||||
if(s && s.contains("window.pageData = ")){
|
||||
def prefix = "window.pageData ="
|
||||
def s1 = s.substring(s.indexOf(prefix)+prefix.size())
|
||||
s1 = s1.substring(0, s1.indexOf("}};")+2)
|
||||
|
||||
JSONObject jsonObject = JSONObject.fromObject(s1);
|
||||
print jsonObject
|
||||
}
|
||||
|
||||
return s
|
||||
|
||||
}
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
checkAbuyun(true);
|
||||
|
||||
def responseMap = [:]
|
||||
// 创建代理服务器地址对象
|
||||
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
|
||||
// 创建HTTP类型代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
|
||||
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
if(content){
|
||||
outputStream.write(content.getBytes());
|
||||
}
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static checkAbuyun(boolean check) {
|
||||
if (check) {
|
||||
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
AbuyunRequestCount.incrementAndGet();
|
||||
} else {
|
||||
AbuyunRequestCount.decrementAndGet();
|
||||
}
|
||||
}
|
||||
}
|
||||
+715
@@ -0,0 +1,715 @@
|
||||
package com.fangjia.script.companyCrawl
|
||||
|
||||
import com.eclipsesource.v8.V8
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import jdk.nashorn.api.scripting.ScriptObjectMirror
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.apache.http.Header
|
||||
import org.apache.http.HttpResponse
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
|
||||
import javax.script.ScriptEngine
|
||||
import javax.script.ScriptEngineManager;
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class GsxtCrawl {
|
||||
|
||||
def static final proxyPort = 9020;
|
||||
def static final proxyServer = "http-dyn.abuyun.com";
|
||||
def static final proxyUser = "HZU1XU28H3G4Z44D";
|
||||
def static final proxyPass = "466C86ED77FE43D4";
|
||||
|
||||
public final static int AbuyunMaxCount = 10;
|
||||
static AtomicInteger AbuyunRequestCount = new AtomicInteger(0);
|
||||
|
||||
def companyKeys = ["亨鑫","捷东","盈复","隆庆","卓源 ","合万","通广","安盛","东利","瑞科 ","凡迪","捷信","安盛","安盛","东进","源盛","茂禄","泰亚","宏成","光益","华圣","益全","长凯","宏新","锦立","宜佳","万美","正欣","和利","国宏","和裕","新昌","佳升","正德","源美","宝峰","金广","龙奥","宏美","成同","义丰","全洪","国聚","亨信","百益","广宏","仁瑞","亿昌","东旭","聚凯","洪成","兴同","赛迪","亨国","浩长","达宏","寿满","亿信","本盛","盈凯","浩汇","元辉","祥生","皇弘","讯思","新州","国泰","成合","茂浩","信源","润凯","虎光","安瑞","益吉","达兴","顺光","发盛","金优","捷丰","春凯","昌兴","本佳","远盛","盛达","亚飞","东昌","益如","宝东","德信","浩盈","多泰","泰联","升聚","辉丰","茂发","盈德","旭嘉","优源","益同","协盈","锦泰","捷久","本宏","德泰","聚富","全庆","凯安","进恒","全伟","优元","凯安","义丰","丰宇","瑞优","隆泰","诺德","台升","裕大","亨恒","义浩","百亨","欣昌","茂全","同亨","长圣","成谦","达泰","高协","正思","安巨","新耀","利仁","国益","旺洪","汇中","旭昌","弘康","优盛","旭达","晶安","华浩","耀春","圣本","康丰","光复","同凌","宇伯","乐鸿","成泰","宏协","华信","达泰","广丰","泰全","汇恒","杰森","天畅","恒通","亚达","庆合","丰圣","德长","茂百","汇久","欣鼎","谦和","厚辉","耀德","瑞隆","多辉","昌茂","成高","益升","安达","兴飞","百进","鼎宝","祥盛","信和","隆顺","亚欣","厚全","博文","讯诺","泰如","亨瑞","凯祥","捷浩","文舟","金庆","成仁","万协","宇中","宏祥","永多","艺峰","美益","创赢","奥龙","德宇","国福","飞利","久欣","如亨","安广","协佳","升捷","盛圣","富凯","长鑫","元卓","荣亨","亨仁","康鑫","信和","百新","干瑞","润万","富庭","信宇","源天","凯佳","正升","裕荣","厚泽","晶富","广新","正多","鼎高","丰汇","永华","兴升","复腾","隆诺","中发","广源","康胜","同进","仁捷","康凯","义洪","晶盛","正祥","裕祥","宇新","洪祥","润岳","义富","盛安","浩易"]
|
||||
static main(args) {
|
||||
def task = new GsxtCrawl();
|
||||
//task.process()
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
getObject()
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("http://www.gsxt.gov.cn/corp-query-homepage.html");
|
||||
get.setHeader("Upgrade-Insecure-Requests", "1");
|
||||
get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
|
||||
|
||||
HttpResponse response = client.execute(get);
|
||||
def __jsluid_h = getJsluid(response)
|
||||
def s = getResponseBodyAsString(response)
|
||||
println (s)
|
||||
def __jsl_clearance = getJsContent(s);
|
||||
|
||||
//Cookie: __jsluid_h=26cb8385493c2e424767eca767720c87; __jsl_clearance=1618899529.627|-1|ao39lP6IVSloPMvq5%2FOSBqSiPZ0%3D
|
||||
def newCookie = __jsluid_h+";"+__jsl_clearance
|
||||
get.setHeader("Cookie", newCookie)
|
||||
HttpResponse response2 = client.execute(get);
|
||||
def s2 = getResponseBodyAsString(response2)
|
||||
def __jsl_clearance2 = getJsContent2(s2);
|
||||
println(__jsl_clearance2)
|
||||
|
||||
// fristCrawl()
|
||||
}
|
||||
|
||||
|
||||
//该函数测试Java与js对象
|
||||
static void getObject() throws Exception{
|
||||
String str = "var _0xbe8c = ['w4LCg8Onw7o=', 'ZAjDvCw=', 'YcKOw6Fm', 'wpHCqsO3CQ==', 'UMKCXnw=', 'w7Aww6XCnQ==', 'BcKNwrvCng==', 'J2zCnsKi', 'wq02WMOx', 'RQXDrUw=', 'eCnCnDw=', 'Q8KowpzDjA==', 'YznDucOn', 'YcOrXxk=', 'w4t9PsKM', 'DMO4QmA=', 'w5HCklPDmQ==', 'w5rCv8Omw6Q=', 'NcOwc8K5', 'GUxjw7Q=', 'w7zDicK8w4I=', 'VCDDpsKv', 'cwLChT4=', 'wpvCl1vDqw==', 'wrvCvMOKNQ==', 'RTnDrAc=', 'GMKFw4hk', 'SxnDvC0=', 'H2NHNQ==', 'w6Aqw7LCqg==', 'wq0mW8Ok', 'L1zCv8Kp', 'B8OqwoI2', 'Jl7DuX8=', 'JcOBwpIR', 'U8OnfSo=', '6K2K5rOG6aqL6KyM', 'f8Kiwo/Dhg==', 'aMK6w6vDog==', 'JcKgw4FC', 'w61vJEg=', 'LMOrwoY0', 'GcK4w6lt', 'w4XDrsO5wpY=', 'J8OhwoM1', 'w5Mzw7J/', 'wocbcBA=', 'SibDt8Oj', 'w7xgwoXDkQ==', 'AcKyDEQ=', 'Rg7Dkw==', 'w6UFK8Kw', 'TTzDjcKK', 'w6QhDcKy', 'ZBLDols=', 'ZjPDrk4=', 'w6XDqX3Ctw==', 'w6EvJcKT', 'JHtUDQ==', 'wq8mI8KE', 'JMOhwoJ9', 'MsK1AkE=', 'w7fCmhcm', 'VsK2w5fDtQ==', 'w7EUw4PCnw==', 'dz/DgsKu', 'UzfDtcK0', 'wpd9fWE=', 'worDvcKFw4M=', 'w4PDr0zDhg==', 'w7ocSMKg', 'wp1uwoZd', 'eAjDv8Oz', 'aATCgTw=', 'w7UGw7k=', 'wqZCwotq', 'HFIBw5s=', 'WsOdw7JB', 'w5PDpXHCpA==', 'GMO8wqdD', 'w4TCsQs=', 'w6/DmcODwpg=', 'w7MuwrjDoQ==', 'dDHDhsOj', 'G308w70=', 'CsO6wog9', 'FcONwrVw', 'M3vCmMK4', 'w6/Dm8OvwpI=', 'QsOLw5ZM', 'TgLDicKy', 'QSHCuQA=', 'UsOlw5RI', 'w7EqwqDDrw==', 'w6jDusK5w4k=', 'OsKaw5rDlg==', 'wosLQsOk', 'DcKxw7Vo', 'w5VuIEo=', 'w7sXw6TCqw==', 'wrLDr8KRw7s=', 'wr9+wpJR', 'w4LDscO0wrQ=', 'FFIPw6U=', 'J8Ksw5rDlA==', 'd8KZwqjDgw==', 'PcKvDF4=', 'MHh2GA==', 'dgDDs0Y=', 'wpxSS1Y=', 'w7l5JMKh', 'w7N8JMKC', 'XsKhw7hR', 'QxDDqm4=', 'PcKGw5ZJ', 'w7HCuAIQ', 'E8Kmw53Dmg==', 'CMOrwok3', 'U8K9wpvDnA==', 'IW06w5U=', 'w7DDjcOGwpQ=', 'X8KlwpHDpA==', 'w6TDn8O1wpo=', 'ZjnDv08=', 'w7/Cqjsp', 'wqLDk8KYw58=', 'w6fDvMKXw4g=', 'FsK8IUY=', 'w4knw4Rh', 'w5zCu8Ogw5M=', 'wqJ0ckc=', 'T8Kgw69G', 'GF0hw74=', 'FMOkwqkH', 'wokmbMOM', 'djHDtcKW', 'FXkow64=', 'B8OHwowH', 'VSjCpQU=', 'w5vDs8K0w4s=', 'w7vCkUV1', 'w7dMIcK8', 'WMOjZyI=', 'LcKBw7po', 'BMK7HkY=', 'B8K0w7F4', 'YsK+w67Drw==', 'HlQKw7I=', 'wpLCrMKrNg==', 'w7IjcsKy', 'w6zDusKhw4o=', 'WhPDm8Om', 'dMKlw7RO', 'dcOHw5R3', 'Nkpww7M=', 'wrnCsMKLdg==', 'dDTDqMKb', 'S8KXZXE=', 'wq/CssOpJA==', 'w7J/KFw=', 'QMKSWEo=', 'woNxbEc=', 'wqVIwow=', 'TD/Dqg4=', 'w7TDpmzCiw==', 'w7HDjWDCgA==', 'w7IEw6vCoA==', 'Q8K+w4zDiQ==', 'PElvw7k=', 'cDXDsRo=', 'FMOLX0c=', 'Z8KHwojDoA==', 'TTTCqzo=', 'woXCrsK6Uw==', 'w4lhHnY=', 'w6Q/dsKj', 'KsKIwpLCtQ==', 'QQTDn8Oy', 'D8Omw7zDoQ==', 'E8KzGk0=']; (function(_0x44c192, _0xbe8c61) {\n" +
|
||||
"\t\tvar _0x1533fb = function(_0x5b499f) {\n" +
|
||||
"\t\t\twhile (--_0x5b499f) {\n" +
|
||||
"\t\t\t\t_0x44c192['push'](_0x44c192['shift']());\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x1533fb(++_0xbe8c61);\n" +
|
||||
"\t} (_0xbe8c, 0x185));\n" +
|
||||
"\tvar _0x1533 = function(_0x44c192, _0xbe8c61) {\n" +
|
||||
"\t\t_0x44c192 = _0x44c192 - 0x0;\n" +
|
||||
"\t\tvar _0x1533fb = _0xbe8c[_0x44c192];\n" +
|
||||
"\t\tif (_0x1533['GlBvhA'] === undefined) { (function() {\n" +
|
||||
"\t\t\t\tvar _0xa131c3 = function() {\n" +
|
||||
"\t\t\t\t\tvar _0x9829e5;\n" +
|
||||
"\t\t\t\t\ttry {\n" +
|
||||
"\t\t\t\t\t\t_0x9829e5 = Function('return\\x20(function()\\x20' + '{}.constructor(\\x22return\\x20this\\x22)(\\x20)' + ');')();\n" +
|
||||
"\t\t\t\t\t} catch(_0x1ba3fa) {\n" +
|
||||
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t\treturn _0x9829e5;\n" +
|
||||
"\t\t\t\t};\n" +
|
||||
"\t\t\t\tvar _0x5e86d6 = _0xa131c3();\n" +
|
||||
"\t\t\t\tvar _0x67fb0c = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=';\n" +
|
||||
"\t\t\t\t_0x5e86d6['atob'] || (_0x5e86d6['atob'] = function(_0x1cb044) {\n" +
|
||||
"\t\t\t\t\tvar _0xa69f93 = String(_0x1cb044)['replace'](/=+\$/, '');\n" +
|
||||
"\t\t\t\t\tvar _0x26bfb9 = '';\n" +
|
||||
"\t\t\t\t\tfor (var _0x2f1d79 = 0x0,\n" +
|
||||
"\t\t\t\t\t_0x3713bb, _0x4e514b, _0x29733b = 0x0; _0x4e514b = _0xa69f93['charAt'](_0x29733b++);~_0x4e514b && (_0x3713bb = _0x2f1d79 % 0x4 ? _0x3713bb * 0x40 + _0x4e514b: _0x4e514b, _0x2f1d79++%0x4) ? _0x26bfb9 += String['fromCharCode'](0xff & _0x3713bb >> ( - 0x2 * _0x2f1d79 & 0x6)) : 0x0) {\n" +
|
||||
"\t\t\t\t\t\t_0x4e514b = _0x67fb0c['indexOf'](_0x4e514b);\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t\treturn _0x26bfb9;\n" +
|
||||
"\t\t\t\t});\n" +
|
||||
"\t\t\t} ());\n" +
|
||||
"\t\t\tvar _0x38a7ff = function(_0x5dd34b, _0x3a0f2e) {\n" +
|
||||
"\t\t\t\tvar _0x3e7a99 = [],\n" +
|
||||
"\t\t\t\t_0x3e2c47 = 0x0,\n" +
|
||||
"\t\t\t\t_0x383663,\n" +
|
||||
"\t\t\t\t_0x14720e = '',\n" +
|
||||
"\t\t\t\t_0x5864f4 = '';\n" +
|
||||
"\t\t\t\t_0x5dd34b = atob(_0x5dd34b);\n" +
|
||||
"\t\t\t\tfor (var _0x28a6d2 = 0x0,\n" +
|
||||
"\t\t\t\t_0x2ea622 = _0x5dd34b['length']; _0x28a6d2 < _0x2ea622; _0x28a6d2++) {\n" +
|
||||
"\t\t\t\t\t_0x5864f4 += '%' + ('00' + _0x5dd34b['charCodeAt'](_0x28a6d2)['toString'](0x10))['slice']( - 0x2);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\t_0x5dd34b = decodeURIComponent(_0x5864f4);\n" +
|
||||
"\t\t\t\tvar _0x3091fb;\n" +
|
||||
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3091fb;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\tfor (_0x3091fb = 0x0; _0x3091fb < 0x100; _0x3091fb++) {\n" +
|
||||
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb] + _0x3a0f2e['charCodeAt'](_0x3091fb % _0x3a0f2e['length'])) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\t_0x3091fb = 0x0;\n" +
|
||||
"\t\t\t\t_0x3e2c47 = 0x0;\n" +
|
||||
"\t\t\t\tfor (var _0x8c68bc = 0x0; _0x8c68bc < _0x5dd34b['length']; _0x8c68bc++) {\n" +
|
||||
"\t\t\t\t\t_0x3091fb = (_0x3091fb + 0x1) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x3e2c47 = (_0x3e2c47 + _0x3e7a99[_0x3091fb]) % 0x100;\n" +
|
||||
"\t\t\t\t\t_0x383663 = _0x3e7a99[_0x3091fb];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3091fb] = _0x3e7a99[_0x3e2c47];\n" +
|
||||
"\t\t\t\t\t_0x3e7a99[_0x3e2c47] = _0x383663;\n" +
|
||||
"\t\t\t\t\t_0x14720e += String['fromCharCode'](_0x5dd34b['charCodeAt'](_0x8c68bc) ^ _0x3e7a99[(_0x3e7a99[_0x3091fb] + _0x3e7a99[_0x3e2c47]) % 0x100]);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\treturn _0x14720e;\n" +
|
||||
"\t\t\t};\n" +
|
||||
"\t\t\t_0x1533['sMWqUz'] = _0x38a7ff;\n" +
|
||||
"\t\t\t_0x1533['AbOzCI'] = {};\n" +
|
||||
"\t\t\t_0x1533['GlBvhA'] = !![];\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x5b499f = _0x1533['AbOzCI'][_0x44c192];\n" +
|
||||
"\t\tif (_0x5b499f === undefined) {\n" +
|
||||
"\t\t\tif (_0x1533['uRkTAI'] === undefined) {\n" +
|
||||
"\t\t\t\t_0x1533['uRkTAI'] = !![];\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x1533fb = _0x1533['sMWqUz'](_0x1533fb, _0xbe8c61);\n" +
|
||||
"\t\t\t_0x1533['AbOzCI'][_0x44c192] = _0x1533fb;\n" +
|
||||
"\t\t} else {\n" +
|
||||
"\t\t\t_0x1533fb = _0x5b499f;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\treturn _0x1533fb;\n" +
|
||||
"\t};\n" +
|
||||
"\tfunction hash(_0x1a5a87) {\n" +
|
||||
"\t\tvar _0x23a3e0 = {};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x93', 'Kp2R') + 'J'] = function(_0x5ebeb2, _0x4681fc) {\n" +
|
||||
"\t\t\treturn _0x5ebeb2 ^ _0x4681fc;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x17', '(bZ%') + 'H'] = function(_0x5108c3, _0x49fe65) {\n" +
|
||||
"\t\t\treturn _0x5108c3 & _0x49fe65;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x88', 'N#(C') + 'T'] = function(_0x3d3f10, _0x3f2a86) {\n" +
|
||||
"\t\t\treturn _0x3d3f10 !== _0x3f2a86;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x60', '(cNt') + 'Z'] = _0x1533('0x70', 'K)PG') + 'J';\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x49', 'Y8u@') + 'h'] = function(_0xc3ff68, _0x129ab0) {\n" +
|
||||
"\t\t\treturn _0xc3ff68 >> _0x129ab0;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x9c', '\$qIi') + 'S'] = function(_0x591dc1, _0x158d9d) {\n" +
|
||||
"\t\t\treturn _0x591dc1 + _0x158d9d;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x7f', '(bZ%') + 'f'] = function(_0xd6deb, _0x557760) {\n" +
|
||||
"\t\t\treturn _0xd6deb >> _0x557760;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x4e', 'S9Bn') + 'd'] = function(_0xd05226, _0x29b6e8) {\n" +
|
||||
"\t\t\treturn _0xd05226 * _0x29b6e8;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x7e', 'y^BN') + 'n'] = function(_0x899f8d, _0x3dbd26) {\n" +
|
||||
"\t\t\treturn _0x899f8d < _0x3dbd26;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x9a', '(U[3') + 'e'] = function(_0x39678a, _0x37d488) {\n" +
|
||||
"\t\t\treturn _0x39678a * _0x37d488;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xa3', 'sahq') + 'U'] = function(_0x4cf877, _0x14e1b0) {\n" +
|
||||
"\t\t\treturn _0x4cf877 << _0x14e1b0;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xa7', 'K#Hh') + 'u'] = function(_0x556336, _0x56b41c) {\n" +
|
||||
"\t\t\treturn _0x556336 - _0x56b41c;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x3d', '(bZ%') + 'p'] = function(_0x372c20, _0x534fe1) {\n" +
|
||||
"\t\t\treturn _0x372c20 >> _0x534fe1;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x1d', 'S9Bn') + 'g'] = function(_0x26a0f8, _0x4f1660) {\n" +
|
||||
"\t\t\treturn _0x26a0f8 >>> _0x4f1660;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0xae', ']*f@') + 'Y'] = function(_0x3423f4, _0x53e1a9) {\n" +
|
||||
"\t\t\treturn _0x3423f4 | _0x53e1a9;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x26', 's4Dp') + 'P'] = function(_0x19382a, _0x220110) {\n" +
|
||||
"\t\t\treturn _0x19382a | _0x220110;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x65', 's4Dp') + 'p'] = function(_0x7b95bc, _0x22e933) {\n" +
|
||||
"\t\t\treturn _0x7b95bc & _0x22e933;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x0', '\$qIi') + 'V'] = function(_0x13269b, _0x227e3b) {\n" +
|
||||
"\t\t\treturn _0x13269b ^ _0x227e3b;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x29', 'J21I') + 'R'] = function(_0x1ba947, _0x4b40e4) {\n" +
|
||||
"\t\t\treturn _0x1ba947(_0x4b40e4);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x50', 's4Dp') + 'n'] = _0x1533('0xa', 'qRqa') + 'e';\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x42', 'ksVw') + 'b'] = function(_0x466fa6, _0x3ce8da) {\n" +
|
||||
"\t\t\treturn _0x466fa6 ^ _0x3ce8da;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x2c', 'rqYb') + 'f'] = function(_0x1e608a, _0x241888) {\n" +
|
||||
"\t\t\treturn _0x1e608a - _0x241888;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x78', 'J21I') + 'u'] = function(_0x564b78, _0x18d236, _0x8fd805) {\n" +
|
||||
"\t\t\treturn _0x564b78(_0x18d236, _0x8fd805);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x69', 'zx(Q') + 'D'] = function(_0x3b7666, _0x47f238, _0x57b05e) {\n" +
|
||||
"\t\t\treturn _0x3b7666(_0x47f238, _0x57b05e);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x5f', 'ZaCV') + 'm'] = function(_0x452fa7, _0x493d76, _0x1446e7, _0x4af814, _0x5e0f59) {\n" +
|
||||
"\t\t\treturn _0x452fa7(_0x493d76, _0x1446e7, _0x4af814, _0x5e0f59);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x22', 'Rh@#') + 'W'] = function(_0x51a858, _0x440b1c) {\n" +
|
||||
"\t\t\treturn _0x51a858(_0x440b1c);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x8', 'xPGU') + 'A'] = function(_0x5e96f2, _0x167b41) {\n" +
|
||||
"\t\t\treturn _0x5e96f2 + _0x167b41;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x6d', 'rqYb') + 'X'] = function(_0x46cd37, _0x5d8684) {\n" +
|
||||
"\t\t\treturn _0x46cd37 + _0x5d8684;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x23a3e0[_0x1533('0x3e', 'rqYb') + 'g'] = function(_0x26bc4b, _0x16bc46) {\n" +
|
||||
"\t\t\treturn _0x26bc4b(_0x16bc46);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tvar _0x179134 = _0x23a3e0;\n" +
|
||||
"\t\tfunction _0x5be8c0(_0x296dd6, _0x778566) {\n" +
|
||||
"\t\t\treturn _0x179134[_0x1533('0x56', 'ZaCV') + 'J'](_0x179134[_0x1533('0x7c', 'o&Q4') + 'H'](_0x296dd6, 0x7fffffff) + (_0x778566 & 0x7fffffff), _0x179134[_0x1533('0x52', 'K)PG') + 'H'](_0x296dd6, 0x80000000)) ^ _0x778566 & 0x80000000;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x225a28(_0x2f57af) {\n" +
|
||||
"\t\t\tvar _0x3c49d8 = {};\n" +
|
||||
"\t\t\t_0x3c49d8[_0x1533('0x55', 'qRqa') + 'N'] = function(_0x15bfb8, _0x51e1eb) {\n" +
|
||||
"\t\t\t\treturn _0x15bfb8 < _0x51e1eb;\n" +
|
||||
"\t\t\t};\n" +
|
||||
"\t\t\tvar _0x567e0f = _0x3c49d8;\n" +
|
||||
"\t\t\tvar _0x5b2e48 = _0x1533('0xa4', 'rQf(') + _0x1533('0x4', 'rqYb') + _0x1533('0x95', 'Rh@#') + _0x1533('0x3', 'OhQr');\n" +
|
||||
"\t\t\tvar _0x321f21 = '';\n" +
|
||||
"\t\t\tfor (var _0x2a5528 = 0x7; _0x2a5528 >= 0x0; _0x2a5528--) {\n" +
|
||||
"\t\t\t\tif (_0x179134[_0x1533('0xb2', 'Gfc!') + 'T'](_0x1533('0x2', 'w*Nn') + 'h', _0x179134[_0x1533('0x6c', 'OhQr') + 'Z'])) {\n" +
|
||||
"\t\t\t\t\t_0x321f21 += _0x5b2e48[_0x1533('0x66', ')T5g') + 'At'](_0x179134[_0x1533('0x9f', 'zx(Q') + 'H'](_0x179134[_0x1533('0x5', 'OhQr') + 'h'](_0x2f57af, _0x2a5528 * 0x4), 0xf));\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\treturn _0x567e0f[_0x1533('0x5a', 'y*wC') + 'N'](t, 0x14) ? 0x5a827999: t < 0x28 ? 0x6ed9eba1: _0x567e0f[_0x1533('0x43', 'zx(Q') + 'N'](t, 0x3c) ? -0x70e44324: -0x359d3e2a;\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\treturn _0x321f21;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0xde630f(_0x10fb9a) {\n" +
|
||||
"\t\t\tvar _0x5b5b6d = _0x179134[_0x1533('0x16', 'GvA9') + 'S'](_0x179134[_0x1533('0x87', 'N#(C') + 'f'](_0x179134[_0x1533('0x83', '[H[u') + 'S'](_0x10fb9a[_0x1533('0x58', 'OhQr') + 'th'], 0x8), 0x6), 0x1),\n" +
|
||||
"\t\t\t_0x797793 = new Array(_0x179134[_0x1533('0x59', 'xPGU') + 'd'](_0x5b5b6d, 0x10));\n" +
|
||||
"\t\t\tfor (var _0x109346 = 0x0; _0x179134[_0x1533('0x61', 'dgE&') + 'n'](_0x109346, _0x179134[_0x1533('0x90', 'y^BN') + 'e'](_0x5b5b6d, 0x10)); _0x109346++) {\n" +
|
||||
"\t\t\t\tif (_0x179134[_0x1533('0x2a', 'NNf8') + 'T'](_0x1533('0x12', 'sahq') + 'y', _0x1533('0x67', 'y*wC') + 'G')) {\n" +
|
||||
"\t\t\t\t\t_0x797793[_0x109346] = 0x0;\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\tfor (_0x109346 = 0x0; _0x109346 < _0x10fb9a[_0x1533('0x40', 'ad*k') + 'th']; _0x109346++) {\n" +
|
||||
"\t\t\t\t_0x797793[_0x109346 >> 0x2] |= _0x179134[_0x1533('0xa9', '@gLq') + 'U'](_0x10fb9a[_0x1533('0x1c', 'GvA9') + _0x1533('0xf', 'OhQr') + 'At'](_0x109346), _0x179134[_0x1533('0xb3', 'S9Bn') + 'u'](0x18, (_0x109346 & 0x3) * 0x8));\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x797793[_0x179134[_0x1533('0xb1', 'qRqa') + 'p'](_0x109346, 0x2)] |= 0x80 << 0x18 - (_0x109346 & 0x3) * 0x8;\n" +
|
||||
"\t\t\t_0x797793[_0x5b5b6d * 0x10 - 0x1] = _0x10fb9a[_0x1533('0xb', '@mqx') + 'th'] * 0x8;\n" +
|
||||
"\t\t\treturn _0x797793;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x50e067(_0x45a1cc, _0x3bcee1) {\n" +
|
||||
"\t\t\treturn _0x45a1cc << _0x3bcee1 | _0x179134[_0x1533('0x5b', 'Y8u@') + 'g'](_0x45a1cc, _0x179134[_0x1533('0x91', '@mqx') + 'u'](0x20, _0x3bcee1));\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x526d3b(_0xb72077, _0xed7f25, _0x5bc269, _0x572b62) {\n" +
|
||||
"\t\t\tif (_0xb72077 < 0x14) return _0x179134[_0x1533('0x98', 'Gfc!') + 'Y'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x68', 'OhQr') + 'H'](~_0xed7f25, _0x572b62));\n" +
|
||||
"\t\t\tif (_0x179134[_0x1533('0x9e', 'w*Nn') + 'n'](_0xb72077, 0x28)) return _0x179134[_0x1533('0x99', ')T5g') + 'J'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
|
||||
"\t\t\tif (_0xb72077 < 0x3c) return _0x179134[_0x1533('0x13', 'CNzQ') + 'P'](_0xed7f25 & _0x5bc269, _0x179134[_0x1533('0x45', '@mqx') + 'p'](_0xed7f25, _0x572b62)) | _0x179134[_0x1533('0xa0', 'avK7') + 'p'](_0x5bc269, _0x572b62);\n" +
|
||||
"\t\t\treturn _0x179134[_0x1533('0x5e', 'avK7') + 'V'](_0xed7f25, _0x5bc269) ^ _0x572b62;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tfunction _0x496785(_0x583c97) {\n" +
|
||||
"\t\t\treturn _0x583c97 < 0x14 ? 0x5a827999: _0x583c97 < 0x28 ? 0x6ed9eba1: _0x583c97 < 0x3c ? -0x70e44324: -0x359d3e2a;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x45b840 = _0x179134[_0x1533('0x39', 'R#BF') + 'R'](_0xde630f, _0x1a5a87);\n" +
|
||||
"\t\tvar _0x53258d = new Array(0x50);\n" +
|
||||
"\t\tvar _0xc86ce6 = 0x67452301;\n" +
|
||||
"\t\tvar _0x3ee50e = -0x10325477;\n" +
|
||||
"\t\tvar _0xdb1008 = -0x67452302;\n" +
|
||||
"\t\tvar _0xe99bcb = 0x10325476;\n" +
|
||||
"\t\tvar _0x56c860 = -0x3c2d1e10;\n" +
|
||||
"\t\tfor (var _0x4c314c = 0x0; _0x4c314c < _0x45b840[_0x1533('0x46', '\$qIi') + 'th']; _0x4c314c += 0x10) {\n" +
|
||||
"\t\t\tvar _0x41e8e8 = _0xc86ce6;\n" +
|
||||
"\t\t\tvar _0x1b7277 = _0x3ee50e;\n" +
|
||||
"\t\t\tvar _0x30f7c3 = _0xdb1008;\n" +
|
||||
"\t\t\tvar _0x279c95 = _0xe99bcb;\n" +
|
||||
"\t\t\tvar _0xf6e888 = _0x56c860;\n" +
|
||||
"\t\t\tfor (var _0x77e31b = 0x0; _0x179134[_0x1533('0x34', 'Y8u@') + 'n'](_0x77e31b, 0x50); _0x77e31b++) {\n" +
|
||||
"\t\t\t\tif (_0x77e31b < 0x10) {\n" +
|
||||
"\t\t\t\t\tif (_0x179134[_0x1533('0xa6', 'Mifp') + 'T'](_0x1533('0x3f', 'o&Q4') + 'G', _0x179134[_0x1533('0xa1', 'rqYb') + 'n'])) {\n" +
|
||||
"\t\t\t\t\t\t_0x53258d[_0x77e31b] = _0x45b840[_0x4c314c + _0x77e31b];\n" +
|
||||
"\t\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\t\tsleepT = _0x179134[_0x1533('0x48', 'NNf8') + 'R'](parseInt, data['wt']) > ret[0x1] ? _0x179134[_0x1533('0x97', 'd[i)') + 'R'](parseInt, data['wt']) - ret[0x1] : 0x1f4;\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t} else {\n" +
|
||||
"\t\t\t\t\t_0x53258d[_0x77e31b] = _0x50e067(_0x179134[_0x1533('0x96', 'rQf(') + 'b'](_0x179134[_0x1533('0xb0', 'Gfc!') + 'b'](_0x53258d[_0x179134[_0x1533('0xa5', 'K)PG') + 'u'](_0x77e31b, 0x3)], _0x53258d[_0x179134[_0x1533('0x51', 'K)PG') + 'f'](_0x77e31b, 0x8)]) ^ _0x53258d[_0x77e31b - 0xe], _0x53258d[_0x179134[_0x1533('0xaf', '(U[3') + 'f'](_0x77e31b, 0x10)]), 0x1);\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t\tt = _0x179134[_0x1533('0x18', 'GvA9') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x23', '\$qIi') + 'u'](_0x5be8c0, _0x179134[_0x1533('0x8d', 'Mifp') + 'D'](_0x50e067, _0xc86ce6, 0x5), _0x179134[_0x1533('0x4c', 'xPGU') + 'm'](_0x526d3b, _0x77e31b, _0x3ee50e, _0xdb1008, _0xe99bcb)), _0x5be8c0(_0x5be8c0(_0x56c860, _0x53258d[_0x77e31b]), _0x179134[_0x1533('0xad', 'rqYb') + 'W'](_0x496785, _0x77e31b)));\n" +
|
||||
"\t\t\t\t_0x56c860 = _0xe99bcb;\n" +
|
||||
"\t\t\t\t_0xe99bcb = _0xdb1008;\n" +
|
||||
"\t\t\t\t_0xdb1008 = _0x50e067(_0x3ee50e, 0x1e);\n" +
|
||||
"\t\t\t\t_0x3ee50e = _0xc86ce6;\n" +
|
||||
"\t\t\t\t_0xc86ce6 = t;\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0xc86ce6 = _0x5be8c0(_0xc86ce6, _0x41e8e8);\n" +
|
||||
"\t\t\t_0x3ee50e = _0x5be8c0(_0x3ee50e, _0x1b7277);\n" +
|
||||
"\t\t\t_0xdb1008 = _0x5be8c0(_0xdb1008, _0x30f7c3);\n" +
|
||||
"\t\t\t_0xe99bcb = _0x5be8c0(_0xe99bcb, _0x279c95);\n" +
|
||||
"\t\t\t_0x56c860 = _0x5be8c0(_0x56c860, _0xf6e888);\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\treturn _0x179134[_0x1533('0x86', 'Gfc!') + 'A'](_0x179134[_0x1533('0x44', 'DDSy') + 'X'](_0x179134[_0x1533('0x8e', 'xPGU') + 'g'](_0x225a28, _0xc86ce6), _0x179134[_0x1533('0x77', 'n*Rs') + 'g'](_0x225a28, _0x3ee50e)) + _0x225a28(_0xdb1008) + _0x225a28(_0xe99bcb), _0x179134[_0x1533('0xd', 'DDSy') + 'g'](_0x225a28, _0x56c860));\n" +
|
||||
"\t}\n" +
|
||||
"\tfunction go(_0x394549) {\n" +
|
||||
"\t\tvar _0x33d0d6 = {};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x2b', 'sahq') + 'H'] = _0x1533('0x14', 'rQf(') + _0x1533('0x15', 'sahq');\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0xab', 'dgE&') + 'H'] = function(_0x23f209, _0x4ed415) {\n" +
|
||||
"\t\t\treturn _0x23f209 < _0x4ed415;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x5c', 'xPGU') + 'g'] = function(_0x39bd0a, _0x4d5764) {\n" +
|
||||
"\t\t\treturn _0x39bd0a != _0x4d5764;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x37', 'y*wC') + 'N'] = function(_0x3c9a26, _0x109b89) {\n" +
|
||||
"\t\t\treturn _0x3c9a26 + _0x109b89;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x63', 'gkl*') + 'z'] = function(_0x584935, _0x58347c) {\n" +
|
||||
"\t\t\treturn _0x584935 + _0x58347c;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x21', 'ZaCV') + 'z'] = function(_0x3fc795, _0x6962f6) {\n" +
|
||||
"\t\t\treturn _0x3fc795 + _0x6962f6;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x27', '(cNt') + 'I'] = function(_0x303446, _0x3388fb) {\n" +
|
||||
"\t\t\treturn _0x303446 == _0x3388fb;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x6b', 'y*wC') + 't'] = function(_0x5cb515, _0x31e7f3) {\n" +
|
||||
"\t\t\treturn _0x5cb515(_0x31e7f3);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x75', 'Rh@#') + 'E'] = function(_0x529d8e, _0xc1423a) {\n" +
|
||||
"\t\t\treturn _0x529d8e + _0xc1423a;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x62', 'rQf(') + 'a'] = function(_0x453505, _0x50cd56) {\n" +
|
||||
"\t\t\treturn _0x453505 + _0x50cd56;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x2e', 'NNf8') + 'B'] = _0x1533('0x28', ']*f@') + _0x1533('0x57', 'Kp2R') + '\\x20/';\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0xe', 'Y8u@') + 'y'] = function(_0x25bcfd, _0x4b634e, _0x1bef22) {\n" +
|
||||
"\t\t\treturn _0x25bcfd(_0x4b634e, _0x1bef22);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x92', 'J21I') + 'i'] = function(_0xccfc4d, _0x9a8620) {\n" +
|
||||
"\t\t\treturn _0xccfc4d > _0x9a8620;\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x11', '@H8e') + 'n'] = function(_0x1b9fc7, _0x7d3ce) {\n" +
|
||||
"\t\t\treturn _0x1b9fc7(_0x7d3ce);\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\t_0x33d0d6[_0x1533('0x1b', 'N#(C') + 'O'] = _0x1533('0x7', 'N#(C') + '失败';\n" +
|
||||
"\t\tvar _0x2eda7d = _0x33d0d6;\n" +
|
||||
"\t\tfunction _0x29e502() {\n" +
|
||||
"\t\t\tvar _0x24a3c1 = window[_0x1533('0x35', 'ad*k') + _0x1533('0x5d', 'Y8u@') + 'r'][_0x1533('0x19', 'avK7') + _0x1533('0x7d', 'cJ3U') + 't'],\n" +
|
||||
"\t\t\t_0x5f1192 = [_0x2eda7d[_0x1533('0x24', '(bZ%') + 'H']];\n" +
|
||||
"\t\t\tfor (var _0x2d8686 = 0x0; _0x2eda7d[_0x1533('0x1a', 'avK7') + 'H'](_0x2d8686, _0x5f1192[_0x1533('0x3b', 'Y8u@') + 'th']); _0x2d8686++) {\n" +
|
||||
"\t\t\t\tif (_0x2eda7d[_0x1533('0x2f', 'y*wC') + 'g'](_0x24a3c1[_0x1533('0x76', 'y*wC') + _0x1533('0x33', 'ZaCV')](_0x5f1192[_0x2d8686]), -0x1)) {\n" +
|
||||
"\t\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\tif (window[_0x1533('0x41', 'dgE&') + _0x1533('0x6e', 'dgE&') + _0x1533('0x85', 'NNf8')] || window[_0x1533('0x30', 'o&Q4') + _0x1533('0x38', 'OhQr')] || window[_0x1533('0xc', 'OhQr') + _0x1533('0x74', 'qRqa')] || window[_0x1533('0x25', '(bZ%') + _0x1533('0x4f', 'avK7') + 'r'][_0x1533('0xa2', 'xPGU') + _0x1533('0x53', ')T5g') + 'r'] || window[_0x1533('0xaa', 'cJ3U') + _0x1533('0x6', 'K#@S') + 'r'][_0x1533('0x8f', 'rqYb') + _0x1533('0xac', '(bZ%') + _0x1533('0x9', 'Rh@#') + _0x1533('0x4d', 'rQf(') + 'e'] || window[_0x1533('0x31', 'N#(C') + _0x1533('0x79', 'dgE&') + 'r'][_0x1533('0x4b', 'ksVw') + _0x1533('0x10', 'gkl*') + _0x1533('0xa8', 'd[i)') + _0x1533('0x89', '\$qIi') + _0x1533('0x84', 's4Dp')]) {\n" +
|
||||
"\t\t\t\treturn !! [];\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tif (_0x29e502()) {\n" +
|
||||
"\t\t\treturn;\n" +
|
||||
"\t\t}\n" +
|
||||
"\t\tvar _0x1642ef = new Date();\n" +
|
||||
"\t\tfunction _0x1a1fd9(_0x55baea, _0x579edb) {\n" +
|
||||
"\t\t\tvar _0x551ac6 = _0x394549[_0x1533('0x4a', 'y*wC') + 's'][_0x1533('0x71', 'K#@S') + 'th'];\n" +
|
||||
"\t\t\tfor (var _0x54ccec = 0x0; _0x2eda7d[_0x1533('0x6f', 'dFiI') + 'H'](_0x54ccec, _0x551ac6); _0x54ccec++) {\n" +
|
||||
"\t\t\t\tfor (var _0x562c2d = 0x0; _0x2eda7d[_0x1533('0x9b', '[H[u') + 'H'](_0x562c2d, _0x551ac6); _0x562c2d++) {\n" +
|
||||
"\t\t\t\t\tvar _0x1b8d8c = _0x2eda7d[_0x1533('0x80', '[H[u') + 'N'](_0x2eda7d[_0x1533('0x3a', 'w*Nn') + 'z'](_0x2eda7d[_0x1533('0x8b', 'cJ3U') + 'z'](_0x579edb[0x0], _0x394549[_0x1533('0x32', 'R#BF') + 's'][_0x1533('0x82', '@mqx') + 'tr'](_0x54ccec, 0x1)), _0x394549[_0x1533('0x1f', '*hmq') + 's'][_0x1533('0x47', '(cNt') + 'tr'](_0x562c2d, 0x1)), _0x579edb[0x1]);\n" +
|
||||
"\t\t\t\t\tif (_0x2eda7d[_0x1533('0x6a', '(bZ%') + 'I'](_0x2eda7d[_0x1533('0x9d', 'Kp2R') + 't'](hash, _0x1b8d8c), _0x55baea)) {\n" +
|
||||
"\t\t\t\t\t\treturn [_0x1b8d8c, new Date() - _0x1642ef];\n" +
|
||||
"\t\t\t\t\t}\n" +
|
||||
"\t\t\t\t}\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t};\n" +
|
||||
"\t\tvar _0x1e73b9 = _0x2eda7d[_0x1533('0x36', 'sahq') + 'y'](_0x1a1fd9, _0x394549['ct'], _0x394549[_0x1533('0x2d', '\$qIi')]);\n" +
|
||||
"\t\tif (_0x1e73b9) {\n" +
|
||||
"\t\t\tvar _0x3b1427;\n" +
|
||||
"\t\t\tif (_0x394549['wt']) {\n" +
|
||||
"\t\t\t\t_0x3b1427 = _0x2eda7d[_0x1533('0x1', 'zx(Q') + 'i'](parseInt(_0x394549['wt']), _0x1e73b9[0x1]) ? _0x2eda7d[_0x1533('0x54', 'avK7') + 'n'](parseInt, _0x394549['wt']) - _0x1e73b9[0x1] : 0x1f4;\n" +
|
||||
"\t\t\t} else {\n" +
|
||||
"\t\t\t\t_0x3b1427 = 0x5dc;\n" +
|
||||
"\t\t\t}\n" +
|
||||
"\t\t\t_0x2eda7d[_0x1533('0x72', 'qRqa') + 'y'](setTimeout,\n" +
|
||||
"\t\t\tfunction() {\n" +
|
||||
"\t\t\t\tdocument[_0x1533('0x20', 'rQf(') + 'ie'] = _0x2eda7d[_0x1533('0x73', 'rQf(') + 'E'](_0x2eda7d[_0x1533('0x3c', 'o&Q4') + 'a'](_0x394549['tn'] + '=' + _0x1e73b9[0x0] + (_0x1533('0x8a', 'ksVw') + _0x1533('0x1e', 'GvA9') + '='), _0x394549['vt']), _0x2eda7d[_0x1533('0x8c', 'Gfc!') + 'B']);\n" +
|
||||
"\t\t\t\tlocation[_0x1533('0x7a', 'sahq')] = location[_0x1533('0x64', 'd[i)') + _0x1533('0x81', '(U[3')] + location[_0x1533('0x94', 'sahq') + 'ch'];\n" +
|
||||
"\t\t\t},\n" +
|
||||
"\t\t\t_0x3b1427);\n" +
|
||||
"\t\t} else {\n" +
|
||||
"\t\t\talert(_0x2eda7d[_0x1533('0x7b', ')T5g') + 'O']);\n" +
|
||||
"\t\t}\n" +
|
||||
"\t};\n" +
|
||||
"\tgo({\n" +
|
||||
"\t\t\"bts\": [\"1618915727.945|0|HM5\", \"VoVNIPqcrvFO13woIZK2RY%3D\"],\n" +
|
||||
"\t\t\"chars\": \"ZDUZR0cDjTEXznMkfgavFO\",\n" +
|
||||
"\t\t\"ct\": \"3049e28877124ad21ca0db55e3af95ff316ceef6\",\n" +
|
||||
"\t\t\"ha\": \"sha1\",\n" +
|
||||
"\t\t\"tn\": \"__jsl_clearance\",\n" +
|
||||
"\t\t\"vt\": \"3600\",\n" +
|
||||
"\t\t\"wt\": \"1500\"\n" +
|
||||
"\t})"
|
||||
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
engine.eval(str);
|
||||
//获取对象
|
||||
System.out.println();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getJsContent(def page){
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
//得到脚本引擎
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
//处理加密js
|
||||
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
|
||||
try {
|
||||
//得到解密后的js
|
||||
js = js.replace("document.cookie","var cookie")
|
||||
js = js.replace("location.href=location.pathname+location.search", "")
|
||||
engine.eval(js);
|
||||
def v = engine.get("cookie")
|
||||
return v.toString().substring(0, v.toString().indexOf(";"))
|
||||
} catch (ScriptException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
def getJsContent2(def page){
|
||||
ScriptEngineManager manager = new ScriptEngineManager();
|
||||
//得到脚本引擎
|
||||
ScriptEngine engine = manager.getEngineByName("javascript");
|
||||
//处理加密js
|
||||
String js = page.trim().replace("<script>", "").replace("</script>", "").replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]}));", "y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]})");
|
||||
try {
|
||||
//得到解密后的js
|
||||
//得到解密后的js
|
||||
String result = (String) engine.eval(js);
|
||||
result = result.substring(result.indexOf("var cd"),result.indexOf("dc+=cd;")+7);
|
||||
result = result.replaceAll("document*.*toLowerCase\\(\\)", "'x'");
|
||||
//得到cookie
|
||||
String jsl = (String) engine.eval(result);
|
||||
return jsl;
|
||||
} catch (ScriptException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null
|
||||
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 通过破解动态JavaScript脚本,
|
||||
* 获取cookie名为 __jsl_clearance的值
|
||||
*
|
||||
* @param body 相应内容(一般为第一次请求获取到的动态js字符串)
|
||||
* @return cookie名为 __jsl_clearance的值
|
||||
*/
|
||||
private static String getJslClearance(String body) {
|
||||
//V8:谷歌开源的运行JavaScript脚本的库. 参数:globalAlias=window, 表示window为全局别名,
|
||||
// 告诉V8在运行JavaScript代码时, 不要从代码里找window的定义.
|
||||
V8 runtime = V8.createV8Runtime("window");
|
||||
//将第一次请求pdf资源时获取到的字符串提取成V8可执行的JavaScript代码
|
||||
body = body.trim()
|
||||
.replace("<script>", "")
|
||||
.replace("</script>", "")
|
||||
.replace("eval(y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)}))",
|
||||
"y.replace(/\\b\\w+\\b/g, function(y){return x[f(y,z)-1]||(\"_\"+y)})");
|
||||
//用V8执行该段代码获取新的动态JavaScript脚本
|
||||
String result = runtime.executeStringScript(body);
|
||||
|
||||
//获取 jsl_clearance 的第一段, 格式形如: 1543915851.312|0|
|
||||
String startStr = "document.cookie='";
|
||||
int i1 = result.indexOf(startStr) + startStr.length();
|
||||
int i2 = result.indexOf("|0|");
|
||||
String cookie1 = result.substring(i1, i2 + 3);
|
||||
/*
|
||||
获取 jsl_clearance 的第二段,格式形如: DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D。
|
||||
主要原理是: 新的动态JavaScript脚本是为浏览器设置cookie, 且cookie名为__jsl_clearance
|
||||
其中第一段值(格式形如:1543915851.312|0|)已经明文写好, 用字符串处理方法即可获取.
|
||||
第二段则是一段JavaScript函数, 需要有V8运行返回,
|
||||
该函数代码需要通过一些字符串定位, 提取出来, 交给V8运行.
|
||||
*/
|
||||
startStr = "|0|'+(function(){";
|
||||
int i3 = result.indexOf(startStr) + startStr.length();
|
||||
int i4 = result.indexOf("})()+';Expires");
|
||||
String code = result.substring(i3, i4).replace(";return", ";");
|
||||
String cookie2 = runtime.executeStringScript(code);
|
||||
|
||||
/*
|
||||
拼接两段字符串, 返回jsl_clearance的完整的值.
|
||||
格式形如: 1543915851.312|0|DW2jqgJO5Bo45yYRKLlFbnqQuD0%3D
|
||||
*/
|
||||
return cookie1 + cookie2;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 将HTTP响应体转换为字符串返回
|
||||
*
|
||||
* @param response HTTP响应
|
||||
* @return 响应体的字符串形式
|
||||
* @throws IOException IO异常
|
||||
*/
|
||||
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
|
||||
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
|
||||
}
|
||||
|
||||
|
||||
|
||||
static String getJsluid(HttpResponse response) {
|
||||
Header header = response.getFirstHeader("set-cookie");
|
||||
String[] split = header.getValue().split(";");
|
||||
for (String s : split) {
|
||||
if (s.contains("__jsluid")) {
|
||||
return s.trim();
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
|
||||
def fristCrawl(){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","www.gsxt.gov.cn")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
// headers.put("Content-Type","application/x-www-form-urlencoded")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
// headers.put("Cookie","__jsluid_h=b4ff8b9b0fd86b6ff99edf27e111a9f2")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
// headers.put("Accept-Encoding","gzip, deflate")
|
||||
|
||||
//String s = doPost("http://www.gsxt.gov.cn/corp-query-homepage.html", null, headers)
|
||||
def s = ABuYunUtil.doGet("http://www.gsxt.gov.cn/corp-query-homepage.html",null, headers, "utf-8")
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
def url = "http://www.gsxt.gov.cn/corp-query-search-1.html "
|
||||
for(def key : companyKeys){
|
||||
// def queryBody = "tab=ent_tab&province=&geetest_challenge=1f31bf4565a8f89b93f4f4b25bb2143a&geetest_validate=9584ff51b3aff476fcac55248ec14459&geetest_seccode=9584ff51b3aff476fcac55248ec14459%7Cjordan&token=115406527&searchword="+ URLEncoder.encode("宏美", "utf-8")
|
||||
def queryBody = "tab=ent_tab&province=&geetest_challenge=daa40fb7efec814ff072b12ee12bb25a&geetest_validate=009a8748b31392ffdfb626fbbdccb4be&geetest_seccode=009a8748b31392ffdfb626fbbdccb4be%7Cjordan&token=115394764&searchword="+ URLEncoder.encode(key, "utf-8")
|
||||
def s = crawl(url, queryBody);
|
||||
if(s){
|
||||
print s
|
||||
parser(s);
|
||||
|
||||
}
|
||||
println s
|
||||
}
|
||||
}
|
||||
|
||||
def parser(content){
|
||||
def document = Jsoup.parse(content);
|
||||
if(document){
|
||||
def detailUrls = []
|
||||
Elements elements = JQueryUtils.select([".search_list_item"], document)
|
||||
if(elements && elements.size()>0){
|
||||
for(def el : elements){
|
||||
detailUrls << el.attr("href")
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
def static crawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","www.gsxt.gov.cn")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","226")
|
||||
headers.put("Cache-Control","max-age=0")
|
||||
headers.put("Upgrade-Insecure-Requests","1")
|
||||
headers.put("Origin","http://www.gsxt.gov.cn")
|
||||
headers.put("Content-Type","application/x-www-form-urlencoded")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36")
|
||||
headers.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9")
|
||||
headers.put("Referer","http://www.gsxt.gov.cn/corp-query-homepage.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate")
|
||||
headers.put("Accept-Language","zh-CN,zh;q=0.9")
|
||||
|
||||
// headers.put("Cookie:"," __jsluid_h=4e2326174f7f9d90ad9ac1f92301f2c9; JSESSIONID=8ac4f9ab894733d7c98c9a3d5c26; SECTOKEN=7446451709198207648; __jsl_clearance=1618823989.806|0|jQc7QRg%2B%2FzcIlz44iNwQg0PjZJg%3D; gsxtBrowseHistory1=%0FS%04%06%1D%04%1D%10SNS%24%26%3B%22%3D%3A71%3A%3B01%3A%219FFDDDDFFDFE%40DDDDEFDECDLE%40L%40L%40GSXS%11%1A%00%1A%15%19%11SNS%E5%91%BD%E6%9F%A3%E5%B9%B6%E6%97%84%E8%AE%AE%E7%AF%95%E5%BA%BE%E5%BA%8E%E8%AF%8A%E7%AF%95%E7%91%B2%E6%9D%BD%E9%98%A4%E5%84%98%E5%8E%8CSXS%11%1A%00%00%0D%04%11SNEEMDXS%02%1D%07%1D%00%00%1D%19%11SNEBELLFGCDL%40FG%09; tlb_cookie=S172.16.12.115")
|
||||
|
||||
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
checkAbuyun(true);
|
||||
|
||||
def responseMap = [:]
|
||||
// 创建代理服务器地址对象
|
||||
InetSocketAddress addr = new InetSocketAddress(proxyServer, proxyPort);
|
||||
// 创建HTTP类型代理对象
|
||||
Proxy proxy = new Proxy(Proxy.Type.HTTP, addr);
|
||||
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
if(content){
|
||||
outputStream.write(content.getBytes());
|
||||
}
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
public static checkAbuyun(boolean check) {
|
||||
if (check) {
|
||||
while (AbuyunRequestCount.get() >= AbuyunMaxCount) {
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
AbuyunRequestCount.incrementAndGet();
|
||||
} else {
|
||||
AbuyunRequestCount.decrementAndGet();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+17
-9
@@ -61,10 +61,10 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
}
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2018-11-01","yyyy-MM-dd"))
|
||||
def startDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2020-01-01","yyyy-MM-dd"))
|
||||
// def startDate = DateUtil.getFirstDayOfMonth(new Date())
|
||||
// def endDate = DateUtil.getFirstDayOfMonth(new Date(), 1)
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2019-12-31","yyyy-MM-dd"))
|
||||
def endDate = DateUtil.getFirstDayOfMonth(DateUtil.parseDate("2021-04-30","yyyy-MM-dd"))
|
||||
|
||||
while(flag){
|
||||
println "开始加载解析数据......"
|
||||
@@ -116,7 +116,7 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
|
||||
count++;
|
||||
}catch (Exception e){
|
||||
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.getMessage()
|
||||
println "处理:【${dbo.city}-${dbo.region}-${dbo.district_name}】发生异常,异常信息为:"+e.printStackTrace()
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
@@ -442,16 +442,21 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
if(regionMap){
|
||||
regionMap.each{k,v->
|
||||
def arrays = k.split("###")
|
||||
if(arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
try{
|
||||
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:region, name:arrays[0],address:arrays[1]]
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}else{//无区域
|
||||
cityMap.each{r,dMap->
|
||||
dMap.each{k,v->
|
||||
def arrays = k.split("###")
|
||||
if(arrays[1] && arrays[1]==keyword){
|
||||
if(arrays && arrays.size()>=2 && arrays[1] && arrays[1]==keyword){
|
||||
districts << [city:city, region:r, name:arrays[0],address:arrays[1]]
|
||||
}
|
||||
}
|
||||
@@ -596,10 +601,13 @@ class DealPriceMathBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def getOpenApiMatch(def city, def region, def name){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, "fuzzy", false)
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
|
||||
if(name){
|
||||
def jsonObject = OpenApiClient.addressMatch(city, name, "property", false, region, null,"fuzzy", false)
|
||||
if(jsonObject && jsonObject.credit>0 ){
|
||||
return [city:jsonObject.city, region:jsonObject.district, name:jsonObject.name, address:jsonObject.address]
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
+9
-9
@@ -121,15 +121,15 @@ class AbcEvaluationRowsBatch extends BaseBatch{
|
||||
houseLoudng.fitment_cost = ret.fitment_charge
|
||||
houseLoudng.car_port = ret.carport
|
||||
houseLoudng.house_height = ret.house_hight
|
||||
houseLoudng.worry_facility" : null,
|
||||
houseLoudng.fitment_age = ret.fitment_age
|
||||
houseLoudng.lighting_ventilation" : null,
|
||||
houseLoudng.build_type = ret.build_type
|
||||
houseLoudng.house_property" : null,
|
||||
houseLoudng.noise" : null,
|
||||
houseLoudng.sight" : null,
|
||||
houseLoudng.fitment_rank" : null,
|
||||
houseLoudng.sideset" : null,
|
||||
// houseLoudng.worry_facility" : null,
|
||||
// houseLoudng.fitment_age = ret.fitment_age
|
||||
// houseLoudng.lighting_ventilation" : null,
|
||||
// houseLoudng.build_type = ret.build_type
|
||||
// houseLoudng.house_property" : null,
|
||||
// houseLoudng.noise" : null,
|
||||
// houseLoudng.sight" : null,
|
||||
// houseLoudng.fitment_rank" : null,
|
||||
// houseLoudng.sideset" : null,
|
||||
|
||||
|
||||
houseLoudng.district_no = district._id
|
||||
|
||||
+1
-1
@@ -352,7 +352,7 @@ class HouseExportBatch extends BaseBatch{
|
||||
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-2020*", queryBuilder);
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "latest_seawater", queryBuilder);
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
def map = houses[k];
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+64
-44
@@ -25,6 +25,7 @@ import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import java.util.zip.GZIPInputStream
|
||||
@@ -36,16 +37,17 @@ class GaoDeMapCrawl {
|
||||
|
||||
public static final def dbUrl = "mongodb://192.168.0.235:27018";
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
def updateLists = []
|
||||
def updateErrorLists = []
|
||||
def flagLists = []
|
||||
def index = 0
|
||||
def prefix = "road_"
|
||||
def prefix = "district_"
|
||||
// def categorys = ["141202", "141203"]//中学,小学
|
||||
// def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
def categorys = ["190301"]//道路相关
|
||||
def categorys = ["120000", "120200","120203","120300","120301","120302","120303","120304"]//住宅相关
|
||||
// def categorys = ["190301"]//道路相关
|
||||
def totalCount = new AtomicLong()
|
||||
|
||||
static main(args) {
|
||||
def task = new GaoDeMapCrawl();
|
||||
@@ -109,26 +111,13 @@ class GaoDeMapCrawl {
|
||||
|
||||
def run(){//,"西安","重庆"
|
||||
// def citys = [ "临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄","莱芜","济南","青岛","威海","烟台","潍坊","淄博"]
|
||||
// def citys = ["北京","广州","深圳","合肥","苏州","西安","重庆"]
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def citys = ["重庆","哈尔滨","沈阳","东莞","南昌","昆明","金华","温州","淄博","湛江","肇庆","宜昌","岳阳","安庆","芜湖","西宁","日照","常德","张家口","北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
// def citys = CommonUtils.findAllCity()
|
||||
def totalCount = 0;
|
||||
for(def city : citys){
|
||||
def cityLations = findAllLation(city);
|
||||
println "开始处理["+city +"]:"+cityLations.size()
|
||||
gaoDeCitySearch(cityLations);
|
||||
if(updateLists.size() > 0){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,39 +134,68 @@ class GaoDeMapCrawl {
|
||||
}
|
||||
|
||||
def gaoDeCitySearch(def cityLocations){
|
||||
try {
|
||||
for(def location : cityLocations){
|
||||
index ++
|
||||
// def bound_gd = location.bound_gd
|
||||
|
||||
// final CountDownLatch latch = new CountDownLatch(cityLocations.size())
|
||||
for(def i=0;i<cityLocations.size();i++){
|
||||
def location = cityLocations[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
// try{
|
||||
// totalCount.addAndGet(1)
|
||||
searching(location)
|
||||
// }catch(Exception e){
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
|
||||
// }
|
||||
// });
|
||||
}
|
||||
// latch.await()
|
||||
|
||||
if(updateLists.size() > 0){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
|
||||
def searching(location){
|
||||
// def bound_gd = location.bound_gd
|
||||
// def lats = [bound_gd[1], bound_gd[3]]
|
||||
// def lngs = [bound_gd[0], bound_gd[2]]
|
||||
// lats.sort()
|
||||
// lngs.sort()
|
||||
// println index
|
||||
// if(lngs[0]<=121.418153 && lngs[1]>=121.418153 && lats[0]<=31.218824 && lats[1]>=31.218824){
|
||||
search(location)
|
||||
search(location)
|
||||
// }
|
||||
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0 && flagLists.size()>1000){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
e.printStackTrace()
|
||||
if(updateLists.size() > 0 && updateLists.size()>1000){
|
||||
println " 正常访问共:"+updateLists.size()
|
||||
updateBatch();
|
||||
}
|
||||
if(updateErrorLists.size() > 0 && updateErrorLists.size()>1000){
|
||||
println " 失败访问共:"+updateErrorLists.size()
|
||||
updateErrorBatch();
|
||||
}
|
||||
|
||||
if(flagLists.size()>0 && flagLists.size()>1000){
|
||||
println "有数据的格子共:"+flagLists.size()
|
||||
updateFlagBath();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def search(location){
|
||||
def bound_gd = location.bound_gd
|
||||
def lats = [bound_gd[1], bound_gd[3]]
|
||||
@@ -198,9 +216,11 @@ class GaoDeMapCrawl {
|
||||
if(m.rets){
|
||||
processData(m.rets);
|
||||
location.count = m.rets.size()
|
||||
location.content = m.toString()
|
||||
flagLists.add(location)
|
||||
}else{
|
||||
location.count = 0
|
||||
location.content = m.toString()
|
||||
flagLists.add(location)
|
||||
}
|
||||
|
||||
@@ -210,7 +230,7 @@ class GaoDeMapCrawl {
|
||||
updateErrorLists.addAll(location)
|
||||
}
|
||||
// update(location);
|
||||
println "[ "+count.get()+" ${index}]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||
println "[ "+totalCount.get()+" ]"+location._id+" "+location.city+" "+location.bound_gd+" flag: "+m.status
|
||||
}
|
||||
|
||||
def processData(rets){
|
||||
@@ -336,7 +356,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : updateLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"status","完成").append("m_date", new Date())
|
||||
def doc = new Document(prefix+"status","完成").append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
@@ -358,7 +378,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : flagLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date())
|
||||
def doc = new Document(prefix+"gd_data_count",location.count).append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
@@ -382,7 +402,7 @@ class GaoDeMapCrawl {
|
||||
for(def location : updateErrorLists){
|
||||
if(location && location._id){
|
||||
Document updateDocument = new Document();
|
||||
def doc = new Document(prefix+"status","失败").append("m_date", new Date())
|
||||
def doc = new Document(prefix+"status","失败").append("m_date", new Date()).append("content", location.content)
|
||||
updateDocument.append('$set', doc);
|
||||
Bson updateBson = Filters.eq("_id", location._id);
|
||||
UpdateOneModel<Document> updateOneModel = new UpdateOneModel<Document>(updateBson, updateDocument);
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
|
||||
class LogStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
static main(args) {
|
||||
def task = new LogStat()
|
||||
task.run()
|
||||
// task.run1()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
ExcelUtils.modify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls", { d ->
|
||||
def city = d['C'].replace("市","")
|
||||
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
|
||||
if(!district){
|
||||
println "sss"
|
||||
}
|
||||
if(d['AT'] && district){
|
||||
|
||||
def startDate2 = DateUtil.add(d['X'], -365)
|
||||
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
|
||||
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
|
||||
|
||||
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
|
||||
def price1 = d['AI']
|
||||
def price2 = d['AL']
|
||||
def price3 = d['AQ']
|
||||
def startPrice = 0
|
||||
def endPrice = 0
|
||||
def lastPrice = 0
|
||||
if(price1){
|
||||
startPrice = (price1 - (price1*0.15)).toInteger()
|
||||
endPrice = (price1 + (price1*0.15)).toInteger()
|
||||
lastPrice = price1
|
||||
}else if(price2){
|
||||
startPrice = (price2 - (price2*0.15)).toInteger()
|
||||
endPrice = (price2 + (price2*0.15)).toInteger()
|
||||
lastPrice = price2
|
||||
}else if(price3){
|
||||
startPrice = (price3 - (price3*0.15)).toInteger()
|
||||
endPrice = (price3 + (price3*0.15)).toInteger()
|
||||
lastPrice = price3
|
||||
}
|
||||
def prices = findHousesByPrices(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
|
||||
if(prices && prices.size()>=3){
|
||||
def houseAvgPrice = (prices.sum()/prices.size()).toInteger()
|
||||
d['AD'] = houseAvgPrice
|
||||
d['AC'] = (houseAvgPrice * d['N']).toInteger()
|
||||
}else{
|
||||
def random = new Random().nextInt()%6
|
||||
def randomAvgPrice = lastPrice*random/100 + lastPrice
|
||||
d['AD'] = randomAvgPrice
|
||||
d['AC'] = (randomAvgPrice * d['N']).toInteger()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\案例信息-待反馈评估过程_V2(无颜色).xls",{ d ->
|
||||
// def calDate = DateUtil.parseDate(d['M'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['N'], d['O'], d['P'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
|
||||
if(d['AA'] && d['AB'] && d['AH']){
|
||||
def city = d['C'].replace("市","")
|
||||
def district = CommonDatautils.findDistrict(city, d['AA'], d['AB'], "district")
|
||||
if(district){
|
||||
|
||||
def alias = []
|
||||
if(district.alias){
|
||||
if(!district.alias.contains(district.name)){
|
||||
district.alias << district.name
|
||||
}
|
||||
alias = district.alias
|
||||
}else{
|
||||
alias << district.name
|
||||
}
|
||||
|
||||
// def startDate = DateUtil.add(d['X'], -90)
|
||||
// def endDate = DateUtil.add(d['X'], 5)
|
||||
// def trades = findTrades(district, startDate, endDate)
|
||||
// if(trades){
|
||||
// d['AJ'] = (int)(trades.sum()/trades.size())
|
||||
// d['AK'] = trades.size()
|
||||
// }
|
||||
//
|
||||
// def monthStartDate = DateUtil.getFirstDayOfMonth(d['X'])
|
||||
// def monthEndDate = DateUtil.getFirstDayOfMonth(DateUtil.parseFormatDate(DateUtil.addMonth(d['X'], 1)))
|
||||
// def monthStart = DateUtil.formatByPattern(monthStartDate,"yyyyMMdd").toInteger()
|
||||
// def monthEnd = DateUtil.formatByPattern(monthEndDate,"yyyyMMdd").toInteger()
|
||||
// def prices = findHouses(city, d['AA'], d['AB'], alias, monthStart, monthEnd)
|
||||
// if(prices){
|
||||
// d['AL'] = (int)(prices.sum()/prices.size())
|
||||
// d['AM'] = prices.size()
|
||||
// }
|
||||
//
|
||||
//
|
||||
// def sDate = DateUtil.formatByPattern(monthStartDate,"yyyyMM").toInteger()
|
||||
// def dbo = findThirdPrice(district.city, district.region, district.name, sDate, "lianjia_esf_price")
|
||||
// if(dbo){
|
||||
// d['AN'] = dbo.lianjia_esf_price
|
||||
// }
|
||||
|
||||
def price = d['AH']
|
||||
def startPrice = (price - (price*0.15)).toInteger()
|
||||
def endPrice = (price + (price*0.15)).toInteger()
|
||||
def startDate2 = DateUtil.add(d['X'], -365)
|
||||
def start = DateUtil.formatByPattern(startDate2,"yyyyMMdd")
|
||||
def end = DateUtil.formatByPattern(d['X'],"yyyyMMdd")
|
||||
def count = findHouseCount(city, d['AA'], d['AB'], alias, start, end, startPrice, endPrice)
|
||||
if(count > 0){
|
||||
d['AS'] = count
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
},0)
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
q.put("source", "链家在线")
|
||||
|
||||
def alias = []
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
alias = dist.alias
|
||||
}else{
|
||||
alias << dist.name
|
||||
}
|
||||
|
||||
if(alias){
|
||||
q.put("district_name", new BasicDBObject('$in', alias))
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lt',endDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def prices = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
prices << d.avg_price
|
||||
}
|
||||
}
|
||||
|
||||
return prices
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def name, def alias, def startDate, def endDate){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}')"
|
||||
def prices = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
prices << data.avg_price
|
||||
})
|
||||
return prices
|
||||
}
|
||||
|
||||
def findHouseCount(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
|
||||
def total = 0
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.c
|
||||
})
|
||||
return total
|
||||
}
|
||||
|
||||
def findHousesByPrices(def city, def region, def name, def alias, def startDate, def endDate, def startprice, def endprice){
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
String sql = "select avg_price from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} and region='${region}' and district_name in ('${aliasStr}') and avg_price>=${startprice} and avg_price<=${endprice}"
|
||||
def prices = []
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
prices << data.avg_price
|
||||
})
|
||||
return prices
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 转义字符串中的特殊字符
|
||||
* 仅过滤Lucene regexps, 其他的勿用
|
||||
* @param keyword
|
||||
* @return
|
||||
*/
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
// def findTestDatas(def city, def value) {
|
||||
// def q = new BasicDBObject()
|
||||
// q.put("user_id", "1007751")
|
||||
// q.put("parameter_map.city", city)
|
||||
// q.put("request_uri", "/property/evaluateForTest")
|
||||
//
|
||||
// def keys = new BasicDBObject()
|
||||
// keys.put("res_map",1)
|
||||
// def cursor = LOG.find(q)
|
||||
// def count = 0
|
||||
// println "总量:"+cursor.count()
|
||||
// def datas = []
|
||||
// for(def d : cursor){
|
||||
// if(d && d.code==200 && d.res_map && d.res_map.code==200){
|
||||
// def resMap = d.res_map.result
|
||||
// if(resMap.calcType==value){
|
||||
// def data = []
|
||||
// def rowMap = [:]
|
||||
// rowMap.city = resMap.city
|
||||
// rowMap.region = resMap.district
|
||||
// rowMap.name = resMap.name
|
||||
// rowMap.calDate = d.parameter_map.calDate
|
||||
// data << rowMap
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// return datas;
|
||||
// }
|
||||
|
||||
// for(def i=0;i<datas.size();i++){
|
||||
// def d = datas[i];
|
||||
// def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd")
|
||||
// def weekend = DateUtils.getFirstDayOfWeek(calDate)
|
||||
// def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend)
|
||||
// if(dbo){
|
||||
// d['E'] = dbo.avg_price
|
||||
// }
|
||||
//
|
||||
// def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district")
|
||||
// if(district){
|
||||
// def startDate = DateUtil.addMonth(calDate, -6)
|
||||
// def start = startDate.replace("-","").toInteger()
|
||||
// def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger()
|
||||
// def alias = []
|
||||
// if(district.alias){
|
||||
// if(!district.alias.contains(district.name)){
|
||||
// district.alias << district.name
|
||||
// }
|
||||
// alias = district.alias
|
||||
// }else{
|
||||
// alias << district.name
|
||||
// }
|
||||
// def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end)
|
||||
// d['G'] = count
|
||||
// }
|
||||
|
||||
def findThirdPrice(def city, def region, def name, def s_date,def fieldName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", name)
|
||||
q.put("s_date", s_date)
|
||||
q.put(fieldName, new BasicDBObject(QueryOperators.EXISTS, true))
|
||||
def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date",-1))
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d;
|
||||
break;
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
}
|
||||
+2
-2
@@ -47,7 +47,7 @@ class YouDaDataParser {
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
|
||||
|
||||
static def statDate = "2021-02-01"
|
||||
static def statDate = "2021-03-01"
|
||||
static main(args) {
|
||||
def task = new YouDaDataParser();
|
||||
// task.newHouseparse();//友达一手房存入mongo
|
||||
@@ -285,7 +285,7 @@ class YouDaDataParser {
|
||||
|
||||
|
||||
def esfParser(){
|
||||
def files = ["D:\\数据\\友达\\2021年3月\\二手房\\查询1.DBF"]
|
||||
def files = ["D:\\数据\\友达\\2021年4月\\二手房\\查询1.DBF"]
|
||||
def datas = []
|
||||
for(def fileName : files){
|
||||
def rows = readDBF(fileName);
|
||||
|
||||
+175
@@ -0,0 +1,175 @@
|
||||
package com.fangjia.script.gj.address
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class AddressParser {
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddressParser();
|
||||
task.run()
|
||||
// task.gaoDeAddress()
|
||||
}
|
||||
|
||||
def gaoDeAddress(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match2(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
count.addAndGet(1)
|
||||
println count.get() +" "+data.district_name
|
||||
match(data._id, data.city, data.region, data.school_address, data.school_name);
|
||||
|
||||
} catch (Exception e) {
|
||||
println "异常"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
|
||||
}
|
||||
|
||||
def match(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
// if(region && !name.contains(region)){
|
||||
// locationMap = matchLocation(city, region + name, "name")
|
||||
// }else{
|
||||
locationMap = matchLocation(city + region, name, "name")
|
||||
// }
|
||||
// if(!locationMap){
|
||||
// locationMap = matchLocation(city, address, "address")
|
||||
// }
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("school_lat1", locationMap.glat)
|
||||
data.put("school_lng1", locationMap.glng)
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def match2(id, city, region, address, name){
|
||||
def locationMap = null;
|
||||
|
||||
locationMap = matchLocation2(city + region, name, "name")
|
||||
|
||||
if(!locationMap){
|
||||
locationMap = matchLocation2(city, address, "address")
|
||||
}
|
||||
if(locationMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("gaode_name", locationMap.name)
|
||||
data.put("gaode_address", locationMap.address)
|
||||
if(locationMap.location){
|
||||
def arrs = locationMap.location.split(",")
|
||||
data.put("school_lat", arrs[1].toDouble())
|
||||
data.put("school_lng", arrs[0].toDouble())
|
||||
}
|
||||
district_school_col.update(new BasicDBObject('_id', id), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
def matchLocation(city, address, cat){
|
||||
|
||||
JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("geocodes")) {
|
||||
def geocodes = gloJson.get("geocodes")
|
||||
def datas = geocodes.get(0)
|
||||
if(datas.formatted_address && cat=="name"){
|
||||
if(datas.formatted_address.contains(address)){
|
||||
def glocation = datas.location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}else{
|
||||
return null
|
||||
}
|
||||
}else{
|
||||
def glocation = geocodes.get(0).location
|
||||
def los = glocation.split(",")
|
||||
map.glng = Double.parseDouble(los[0].toString())
|
||||
map.glat = Double.parseDouble(los[1].toString())
|
||||
}
|
||||
}
|
||||
return map
|
||||
}
|
||||
|
||||
|
||||
def matchLocation2(city, address, cat){
|
||||
|
||||
//JSONObject gloJson = GaoDeApiUtils.geo(city, address)
|
||||
JSONObject gloJson = GaoDeApiUtils.search(address, city, '140000')
|
||||
def map = [:]
|
||||
if (gloJson && gloJson.get("pois")) {
|
||||
def geocodes = gloJson.get("pois")
|
||||
def datas = geocodes.get(0)
|
||||
return datas
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("school_name","苏州市娄葑实验小学")
|
||||
// q.put("school_lng", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+354
@@ -0,0 +1,354 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class DistrictSchoolDataExport {
|
||||
|
||||
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def regionCodeMap = [:]
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictSchoolDataExport()
|
||||
// task.run()
|
||||
task.megre()
|
||||
}
|
||||
|
||||
def megre(){
|
||||
def datas1 = ExcelUtils.read("d:\\数据\\楼盘对口学校_V2.xlsx",0,0)
|
||||
def ids = []
|
||||
for(def d : datas1){
|
||||
ids << d['U']
|
||||
}
|
||||
def datas2 = ExcelUtils.read("d:\\数据\\无学区房数据(1).xlsx",0,1)
|
||||
for(def d : datas2){
|
||||
def id = d['U']
|
||||
if(!ids.contains(id)){
|
||||
datas1 << d
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校V1.xlsx", datas1, "数据")
|
||||
}
|
||||
|
||||
def run(){
|
||||
def city = "苏州"
|
||||
getRegionCodeByCity(city)
|
||||
getRegionCodeByCity("合肥")
|
||||
getRegionCodeByCity("西安")
|
||||
def districts0 = CommonDatautils.findAllDistricts("苏州", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts1 = CommonDatautils.findAllDistricts("合肥", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts2 = CommonDatautils.findAllDistricts("西安", "district", ["city", "region", "name", "address", "lng", "lat", "c_date","m_date"]);
|
||||
def districts = []
|
||||
districts.addAll(districts0)
|
||||
districts.addAll(districts1)
|
||||
districts.addAll(districts2)
|
||||
def totalCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
def rets = []
|
||||
rets << ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站","小区ID","数据类型"]
|
||||
def i = 0
|
||||
for(def k=0;k<districts.size();k++){
|
||||
final def dist = districts.get(k)
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
|
||||
def primarySchools = findPrimarySchool(dist.city, dist.region, dist.name)
|
||||
def midSchools = findMiddleSchool(dist.city, dist.region, dist.name)
|
||||
println totalCount.get() +" "+dist.name +" 小学:"+primarySchools.size()+" 初中:"+midSchools.size()
|
||||
if(primarySchools || midSchools){
|
||||
def info1 = null;
|
||||
def info2 = null;
|
||||
def distance1 = 0
|
||||
def distance2 = 0
|
||||
if(primarySchools && dist.lng){
|
||||
info1 = selectSchool(dist.lng/10000000000.0, dist.lat/10000000000.0, primarySchools);
|
||||
if(info1 && info1.school_lat2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance1 = GeoHelper.GetDistance(lat, lng, info1.school_lat2.toDouble(), info1.school_lng2.toDouble())
|
||||
if(distance1>3000){
|
||||
info1 = null
|
||||
}
|
||||
}else{
|
||||
info1 = null
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(midSchools) {
|
||||
info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools);
|
||||
if(info2 && info2.school_lng2){
|
||||
double lat = dist.lat/10000000000.0
|
||||
double lng = dist.lng/10000000000.0
|
||||
distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2)
|
||||
if(distance2>3000){
|
||||
info2 = null
|
||||
}
|
||||
}else{
|
||||
info2 = null
|
||||
}
|
||||
}
|
||||
if(info1|| info2){
|
||||
def row = []
|
||||
row << cityCodeMap.get(dist.city)
|
||||
row << regionCodeMap.get(dist.city +"-"+dist.region)
|
||||
row << dist.name
|
||||
row << dist.lng/10000000000.0
|
||||
row << dist.lat/10000000000.0
|
||||
row << dist.address
|
||||
if(info1){
|
||||
row << info1.school_name
|
||||
row << info1.school_address!=null?info1.school_address:''
|
||||
row << info1.attribute!=null?info1.attribute:''
|
||||
row << info1.school_lng2!=null?info1.school_lng2:''
|
||||
row << info1.school_lat2!=null?info1.school_lat2:''
|
||||
// row << distance1
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
|
||||
if(info2){
|
||||
row << info2.school_name
|
||||
row << info2.school_address!=null?info2.school_address:''
|
||||
row << info2.attribute!=null?info2.attribute:''
|
||||
row << info2.school_lng2!=null?info2.school_lng2:''
|
||||
row << info2.school_lat2!=null?info2.school_lat2:''
|
||||
// row << distance2
|
||||
}else{
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
row << ""
|
||||
// row << ""
|
||||
}
|
||||
row << ""
|
||||
row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss")
|
||||
row << "fjw"
|
||||
row << dist._id.toString()
|
||||
row << "对口"
|
||||
|
||||
rets << row
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据")
|
||||
|
||||
}
|
||||
|
||||
|
||||
def selectSchool(lng, lat, def schools){
|
||||
def minSchool = null
|
||||
def level0 = []//国家
|
||||
def level1 = []//省重点
|
||||
def level2 = []//市重点
|
||||
def level3 = []//区重点
|
||||
def level4 = []//普通
|
||||
def level5 = []//其它
|
||||
for(def s : schools){
|
||||
if(s.attribute){
|
||||
if(s.attribute.contains("国家重点")){
|
||||
level0 << s
|
||||
}else if(s.attribute.contains("省重点")){
|
||||
level1 << s
|
||||
}else if(s.attribute.contains("市重点")){
|
||||
level2 << s
|
||||
}else if(s.attribute.contains("区重点")){
|
||||
level3 << s
|
||||
}else if(s.attribute.contains("普通")){
|
||||
level4 << s
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}else{
|
||||
level5 << s
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if(level0){
|
||||
if(level0.size()==1){
|
||||
minSchool = level0.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level0)
|
||||
}
|
||||
}else if(level1){
|
||||
if(level1.size()==1){
|
||||
minSchool = level1.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level1)
|
||||
}
|
||||
}else if(level2){
|
||||
if(level2.size()==1){
|
||||
minSchool = level2.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level2)
|
||||
}
|
||||
}else if(level3){
|
||||
if(level3.size()==1){
|
||||
minSchool = level3.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level3)
|
||||
}
|
||||
}else if(level4){
|
||||
if(level4.size()==1){
|
||||
minSchool = level4.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level4)
|
||||
}
|
||||
}else if(level5){
|
||||
if(level5.size()==1){
|
||||
minSchool = level5.get(0)
|
||||
}else{
|
||||
minSchool = selectMinDistanceSchool(lat, lng, level5)
|
||||
}
|
||||
}
|
||||
|
||||
def rets = []
|
||||
|
||||
if(minSchool){
|
||||
rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name)
|
||||
}
|
||||
|
||||
if(rets.size()>0){
|
||||
for(def d : rets){
|
||||
if(!minSchool.school_address){
|
||||
minSchool.school_address = d.address
|
||||
}
|
||||
if(!minSchool.attribute){
|
||||
minSchool.attribute = d.attribute
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(minSchool){
|
||||
if(minSchool.school_map_lng){
|
||||
minSchool.school_lng2 = minSchool.school_map_lng
|
||||
minSchool.school_lat2 = minSchool.school_map_lat
|
||||
}else if(minSchool.school_lng1){
|
||||
minSchool.school_lng2 = minSchool.school_lng1
|
||||
minSchool.school_lat2 = minSchool.school_lat1
|
||||
}else if(minSchool.school_lng){
|
||||
minSchool.school_lng2 = minSchool.school_lng
|
||||
minSchool.school_lat2 = minSchool.school_lat
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
return minSchool;
|
||||
}
|
||||
|
||||
|
||||
def findSchoolInfo(def city, def region, def schoolName){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("name", schoolName)
|
||||
def dbcursor = school_col.find(q)
|
||||
def rets = []
|
||||
for(def obj : dbcursor){
|
||||
rets << obj
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
def selectMinDistanceSchool(def lat, def lng, def schools){
|
||||
def minSchool = null;
|
||||
def minDistance = Integer.MAX_VALUE
|
||||
for(def s1 : schools){
|
||||
|
||||
def lng2 = null
|
||||
def lat2 = null
|
||||
if(s1.school_map_lng){
|
||||
lng2 = s1.school_map_lng
|
||||
lat2 = s1.school_map_lat
|
||||
}else if(s1.school_lng1){
|
||||
lng2 = s1.school_lng1
|
||||
lat2 = s1.school_lat1
|
||||
}else if(s1.school_lng){
|
||||
lng2 = s1.school_lng
|
||||
lat2 = s1.school_lat
|
||||
}
|
||||
if(lat2){
|
||||
def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2)
|
||||
println s1.district_name +" "+ s1.school_name+" "+distance
|
||||
if(distance<minDistance){
|
||||
minDistance = distance
|
||||
minSchool = s1
|
||||
minSchool.school_lng2 = lng2
|
||||
minSchool.school_lat2 = lat2
|
||||
}
|
||||
}
|
||||
}
|
||||
return minSchool
|
||||
}
|
||||
|
||||
def findPrimarySchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", Pattern.compile("^.*小学.*\$", Pattern.CASE_INSENSITIVE))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
|
||||
def findMiddleSchool(def city, def region, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_city", city)
|
||||
q.put("match_region", region)
|
||||
q.put("match_name", name)
|
||||
q.put("category", new BasicDBObject(QueryOperators.NE, "小学"))
|
||||
def dbcursor = district_school_col.find(q).sort(new BasicDBObject("attribute", 1))
|
||||
def datas = []
|
||||
for(def db : dbcursor){
|
||||
datas << db
|
||||
}
|
||||
return datas;
|
||||
}
|
||||
|
||||
def getRegionCodeByCity(def city){
|
||||
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
|
||||
for(def d : datas){
|
||||
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,7 +11,7 @@ class FieldStat {
|
||||
}
|
||||
|
||||
def run(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行\\小区融合结果概况2.xlsx", { row ->
|
||||
ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\建行数据融合结果统计_0425.xlsx", { row ->
|
||||
def city = row['A']
|
||||
if(city){
|
||||
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "alias"]);
|
||||
@@ -26,15 +26,14 @@ class FieldStat {
|
||||
aliasTotalCount = aliasTotalCount + alias.size()
|
||||
}
|
||||
}
|
||||
row['J'] = aliasTotalCount
|
||||
row['K'] = aliasIndex
|
||||
// row['J'] = aliasTotalCount
|
||||
// row['K'] = aliasIndex
|
||||
def p = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, aliasIndex*100/total)
|
||||
row['L'] = p+"%"
|
||||
row['M'] = total
|
||||
row['B'] = p+"%"
|
||||
// row['M'] = total
|
||||
println city +" "+p
|
||||
}
|
||||
|
||||
});
|
||||
}, 1);
|
||||
println "finish"
|
||||
}
|
||||
}
|
||||
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
package com.fangjia.script.gj.data
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class RegionPriceExport {
|
||||
|
||||
DBCollection REGION_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "region_sell_stat", "mongo.db.fangjia.com")
|
||||
static main(args) {
|
||||
def task = new RegionPriceExport();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def months = [202010,202011,202012,202101,202102,202103]
|
||||
def regions = CommonUtils.getAllRegions("福州","region")
|
||||
def rets = []
|
||||
rets << ["城市","区域","202010","202011","202012","202101","202102","202103"]
|
||||
regions.each{region,values->
|
||||
def data = []
|
||||
data << "福州"
|
||||
data << region
|
||||
for(def month :months){
|
||||
def dbo = findPrice("福州", region, month);
|
||||
if(dbo){
|
||||
data << dbo.avg_price
|
||||
}else{
|
||||
data << ""
|
||||
}
|
||||
}
|
||||
rets << data
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\福州\\福州区域月均价数据.xlsx",rets, "价格")
|
||||
}
|
||||
|
||||
def findPrice(city, region, month){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("date", month)
|
||||
def dbo = REGION_SELL_STAT.findOne(q)
|
||||
return dbo
|
||||
}
|
||||
}
|
||||
+301
File diff suppressed because one or more lines are too long
@@ -0,0 +1,580 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class CityStat {
|
||||
|
||||
public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", lineUrl)
|
||||
|
||||
public static final DBCollection BAD_CASE_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "bad_case", lineUrl)
|
||||
|
||||
public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
static main(args) {
|
||||
def task = new CityStat()
|
||||
// task.run()
|
||||
// task.run1();
|
||||
task.run3();
|
||||
// task.run4()
|
||||
// task.run6()
|
||||
// task.run7()
|
||||
// task.run8()
|
||||
}
|
||||
|
||||
// def run8(){
|
||||
// ExcelUtils.multiThreadingModify("D:\\数据\\深圳建行\\建行匹配小区(3).xlsx",{d->
|
||||
//
|
||||
// def dist = CommonDatautils.findDistrict(d['N'], d['O'], d['P'], "district")
|
||||
// if(dist){
|
||||
// def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
// }
|
||||
//
|
||||
// },1, 0)
|
||||
// }
|
||||
|
||||
def run5(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
|
||||
def total = 0
|
||||
def scount = 0
|
||||
def index = 0
|
||||
// final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
|
||||
for(def dist : dists){
|
||||
index++
|
||||
def rets = findTradeDetails(dist, "2020-03-24");
|
||||
println index
|
||||
if(rets){
|
||||
total = total + rets.size()
|
||||
def count = findSimialPriceCount(dist, rets);
|
||||
scount = scount+count
|
||||
}
|
||||
}
|
||||
println scount +" "+ total
|
||||
def pre = DataHandleUtils.getDoubleRound(2, scount*100/total)
|
||||
println pre;
|
||||
}
|
||||
|
||||
|
||||
def run7(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("镇江","district",["city", "region","name","alias"])
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist : dists){
|
||||
if(dist.alias) {
|
||||
|
||||
if (!dist.alias.contains(dist.name)) {
|
||||
dist.alias << dist.name
|
||||
}
|
||||
for(def name : dist.alias){
|
||||
distMap.put( dist.region +"#"+name, dist)
|
||||
}
|
||||
}else{
|
||||
distMap.put( dist.region +"#"+dist.name, dist)
|
||||
}
|
||||
}
|
||||
def rets = findTradeDetails2("镇江","2020-03-24");
|
||||
def total = 0
|
||||
def scount = 0
|
||||
|
||||
def index = 0
|
||||
for(def t : rets){
|
||||
def key = t.region+"#"+t.district_name
|
||||
def dist = distMap.get(key)
|
||||
index++
|
||||
println index
|
||||
if(dist){
|
||||
total++
|
||||
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
|
||||
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
|
||||
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
|
||||
if(t.avg_price && weekendPrice){
|
||||
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
|
||||
def m = Math.abs(mod)
|
||||
if(m<=15){
|
||||
scount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println scount +" "+total
|
||||
|
||||
}
|
||||
|
||||
def run6(){
|
||||
def dists = CommonDatautils.findAllDistrictByCity("常州","district",["city", "region","name","alias"])
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
def totalCount = new AtomicLong()
|
||||
def tradeTotal = new AtomicLong()
|
||||
def scount = new AtomicLong()
|
||||
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
def rets = findTradeDetails(dist, "2020-03-24");
|
||||
if(rets){
|
||||
// tradeTotal = tradeTotal + rets.size()
|
||||
tradeTotal.addAndGet(rets.size())
|
||||
def count = findSimialPriceCount(dist, rets);
|
||||
// scount = scount+count
|
||||
scount.addAndGet(count);
|
||||
}
|
||||
println totalCount.get() +" "+dist.name
|
||||
}catch (Exception e){
|
||||
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
println scount +" "+ tradeTotal
|
||||
def pre = DataHandleUtils.getDoubleRound(2, scount*100/tradeTotal)
|
||||
println pre;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def findSimialPriceCount(def dist, def rets){
|
||||
def count = 0
|
||||
for(def t : rets){
|
||||
|
||||
def startDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),-7)
|
||||
def endDate = DateUtils.add(DateUtils.formatToYYYYMMDD(t.trade_date),7)
|
||||
def weekendPrice = CommonDatautils.queryDistrictPriceByDate(dist.city, dist.region, dist.name, startDate, endDate)
|
||||
if(t.avg_price && weekendPrice){
|
||||
def mod = (weekendPrice - t.avg_price)*100/t.avg_price
|
||||
def m = Math.abs(mod)
|
||||
if(m<=15){
|
||||
count++;
|
||||
}
|
||||
}
|
||||
// def weekend = DateUtils.getFirstDayOfWeek()
|
||||
// def stat = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name, weekend)
|
||||
// if(t.avg_price && stat){
|
||||
// def mod = (stat.avg_price - t.avg_price)*100/t.avg_price
|
||||
// def m = Math.abs(mod)
|
||||
// if(m<=15){
|
||||
// count++;
|
||||
// }
|
||||
// }
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
def run3(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\建行银行\\建行匹配小区17.xlsx",{data->
|
||||
if(data['A']){
|
||||
def startDate = DateUtils.addMonth(data['D'], -3)
|
||||
def endDate = DateUtils.addMonth(data['D'], 3)
|
||||
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
|
||||
data['H'] = tradeAvgPrice.toInteger()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
def run4(){
|
||||
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
|
||||
def startDate = DateUtils.addMonth(new Date(), -3)
|
||||
def endDate = "2021-02-01"
|
||||
def result = []
|
||||
def allDistricts = []
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
|
||||
allDistricts.addAll(districts)
|
||||
println city +" "+districts.size()
|
||||
|
||||
}
|
||||
def rets = findDistrictTrade(allDistricts, startDate,endDate);
|
||||
println rets.size()
|
||||
|
||||
|
||||
}
|
||||
|
||||
def findDistrictTrade(def districts, def startDate, def endDate){
|
||||
def rets = []
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
for(def i=0;i<districts.size;i++){
|
||||
def dist = districts.get(i)
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
println count_.incrementAndGet()+" "+dist.city+" "+dist.region+" "+dist.name+" "
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
if(tradeAvgPrice){
|
||||
rets << [dist.city, dist.region, dist.name, tradeAvgPrice.toInteger()]
|
||||
}
|
||||
}catch (Exception e){
|
||||
|
||||
}finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
}
|
||||
latch.await()
|
||||
ExcelUtils.write("D:\\数据\\深圳建行\\成交数据.xlsx",rets, "成交")
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def run(){
|
||||
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
|
||||
def citys = ["上海","北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"]
|
||||
def rets = []
|
||||
rets << ["城市","小区数量","均价覆盖率","样本计算小区均价覆盖率","badcase数量", "近三个月成交数据量","近三个月有成交数据的小区覆盖率"]
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name", "alias"])
|
||||
def priceCount = findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), null);
|
||||
def houseSatCount = 0//findStatCount(city, DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"), 2);
|
||||
|
||||
def badcaseCount = findBadCaseCount(city, DateUtils.parseFromTightByPattern("2020-11-29", "yyyy-MM-dd"));
|
||||
def tradeCount = 0//findDealPriceCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"));
|
||||
|
||||
// def tradeDistCount = calTradeDistCount(city, DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
|
||||
|
||||
def pricePercent = DataHandleUtils.getDoubleRound(2, priceCount*100/districts.size())
|
||||
def houseStatPercent = 0//DataHandleUtils.getDoubleRound(2, houseSatCount*100/districts.size())
|
||||
def badcasePercent = DataHandleUtils.getDoubleRound(2, badcaseCount*100/districts.size())
|
||||
def tradeDistPercent = 0//DataHandleUtils.getDoubleRound(2, tradeDistCount*100/districts.size())
|
||||
|
||||
rets << [city, districts.size(), pricePercent+"%", houseStatPercent+"%", badcasePercent+"%", tradeCount, tradeDistPercent+"%"]
|
||||
println city +" 完成"
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\统计\\城市统计1.xlsx", rets, "统计")
|
||||
}
|
||||
|
||||
def run1(){
|
||||
//"北京","大连","厦门","深圳","杭州","成都","重庆","苏州","广州"
|
||||
def citys = ["上海"]
|
||||
def rets = []
|
||||
rets << ["城市","区域","小区名","成交价","评估价"]
|
||||
for(def city : citys) {
|
||||
def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region", "name", "alias"])
|
||||
def retMap = calTradeDistDetail(city,DateUtils.parseFromTightByPattern("2020-10-01", "yyyy-MM-dd"), districts);
|
||||
retMap.each {k,v->
|
||||
if(v && v.size()>0){
|
||||
def sum = v.sum()
|
||||
def tradeAvgPrice = sum/v.size()
|
||||
def arrs = k.split("#")
|
||||
def stat = findDistrictStat(city, arrs[0], arrs[1], DateUtils.parseFromTightByPattern("2021-01-17", "yyyy-MM-dd"))
|
||||
rets << [city, arrs[0], arrs[1], tradeAvgPrice.toInteger(), stat.avg_price]
|
||||
}
|
||||
}
|
||||
println city +" 完成"
|
||||
}
|
||||
ExcelUtils.write("d:\\数据\\统计\\城市成交数据.xlsx", rets, "成交")
|
||||
}
|
||||
|
||||
def run2(){
|
||||
def datas = ExcelUtils.read("D:\\数据\\深圳建行\\建行匹配小区2.xlsx",0,1)
|
||||
def cityDataMap = [:]
|
||||
def index = 0
|
||||
for(def data : datas){
|
||||
index++
|
||||
println index+" "+data
|
||||
def lists = cityDataMap.get(data['A'])
|
||||
if(!lists){
|
||||
lists = []
|
||||
cityDataMap.put(data['A'], lists)
|
||||
}
|
||||
def dist = CommonDatautils.findDistrict(data['A'], data['B'], data['C'], "district")
|
||||
dist.calDate = data['D']
|
||||
dist.evaluationPrice = data['E']
|
||||
lists << dist
|
||||
|
||||
}
|
||||
def citys = [ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ]
|
||||
def results = []
|
||||
results << ["城市","区域","小区名","成交价","评估价"]
|
||||
for(def city : citys){
|
||||
def districts = cityDataMap.get(city)
|
||||
def rets = findAvgPriceTrades(city, districts)
|
||||
results.addAll(rets)
|
||||
}
|
||||
ExcelUtils.write("D:\\数据\\深圳建行\\建行成交3.xlsx",results, "数据")
|
||||
}
|
||||
|
||||
def findAvgPriceTrades(city, districts){
|
||||
def rets = []
|
||||
final CountDownLatch latch = new CountDownLatch(districts.size())
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def dist = districts[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
def startDate = DateUtils.addMonth(dist.calDate, -3)
|
||||
def endDate = DateUtils.addMonth(dist.calDate, 3)
|
||||
def tradeAvgPrice = findTrades(dist, startDate, endDate);
|
||||
println dist.city +" "+dist.region+" "+dist.name +" "+tradeAvgPrice.toInteger()
|
||||
// def stat = findDistrictStat(city, dist.region, dist.name, DateUtils.parseFromTightByPattern("2021-01-10", "yyyy-MM-dd"))
|
||||
rets << [city, dist.region, dist.name, tradeAvgPrice.toInteger(), dist.evaluationPrice]
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
latch.await()
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findTrades(dist, startDate, endDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt',DateUtils.parseFromTightYYYYMMDD(endDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
def avg_price = 0
|
||||
def total_price = 0
|
||||
def count = 0
|
||||
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
total_price = total_price + d.avg_price
|
||||
count++
|
||||
}
|
||||
}
|
||||
if(count > 3){
|
||||
avg_price = total_price/count
|
||||
}
|
||||
return avg_price
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTradeDetails(dist, startDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", dist.city)
|
||||
q.put("region", dist.region)
|
||||
|
||||
if(dist.alias){
|
||||
if(!dist.alias.contains(dist.name)){
|
||||
dist.alias << dist.name
|
||||
}
|
||||
q.put("district_name", new BasicDBObject('$in', dist.alias))
|
||||
}else{
|
||||
q.put("district_name", dist.name)
|
||||
}
|
||||
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def rets = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findTradeDetails2(city, startDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def rets = []
|
||||
for(def d : dbcorsor){
|
||||
if(d.avg_price){
|
||||
rets << d
|
||||
}
|
||||
}
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findStatCount(def city, def weekend, def value_type){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
if(value_type){
|
||||
q.put("value_type", value_type)
|
||||
}
|
||||
def count = DIST_STAT_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDistrictStat(def city, def region, def name, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("region", region)
|
||||
q.put("district_name", name)
|
||||
q.put("weekend", weekend)
|
||||
def obj = DIST_STAT_COLL.findOne(q)
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findBadCaseCount(def city, def weekend){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def count = BAD_CASE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def findDealPriceCount(def city, def tradeDate){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def count = DEAL_PRICE_COLL.count(q)
|
||||
return count;
|
||||
}
|
||||
|
||||
def calTradeDistCount(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("source","链家在线")
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def rets = []
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist && !rets.contains(dist)){
|
||||
rets << dist
|
||||
}
|
||||
}
|
||||
return rets.size();
|
||||
}
|
||||
|
||||
def findIsExistsDB(def region, def district_name, def distMap){
|
||||
def dist = null;
|
||||
def regionDists = distMap.get(region)
|
||||
for(def rd : regionDists){
|
||||
def alias = rd.alias
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(rd.name)){
|
||||
alias << rd.name
|
||||
}
|
||||
if(alias.contains(district_name)){
|
||||
dist = rd
|
||||
break;
|
||||
}
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
|
||||
def calTradeDistDetail(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist){
|
||||
def tradePrices = retMap.get(dist.region+"#"+dist.name)
|
||||
if(!tradePrices){
|
||||
tradePrices = []
|
||||
retMap.put(dist.region+"#"+dist.name, tradePrices)
|
||||
}
|
||||
if(obj.avg_price){
|
||||
tradePrices << obj.avg_price.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
|
||||
|
||||
def calTradeDistDetail2(def city, def tradeDate, def districts){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("trade_date", new BasicDBObject('$gte', tradeDate))
|
||||
def dbcorsor = DEAL_PRICE_COLL.find(q)
|
||||
|
||||
def distMap = [:]
|
||||
for(def dist :districts){
|
||||
def lists = distMap.get(dist.region)
|
||||
if(!lists){
|
||||
lists = []
|
||||
distMap.put(dist.region, lists);
|
||||
}
|
||||
lists << dist
|
||||
}
|
||||
|
||||
def retMap = [:]
|
||||
for(def obj : dbcorsor){
|
||||
def dist = findIsExistsDB(obj.region, obj.district_name, distMap);
|
||||
if(dist){
|
||||
def tradePrices = retMap.get(dist.region+"#"+dist.name)
|
||||
if(!tradePrices){
|
||||
tradePrices = []
|
||||
retMap.put(dist.region+"#"+dist.name, tradePrices)
|
||||
}
|
||||
if(obj.avg_price){
|
||||
tradePrices << obj.avg_price.toInteger()
|
||||
}
|
||||
}
|
||||
}
|
||||
return retMap;
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.gj.paimai.GaoDeUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CompanyAddressCompare {
|
||||
static main(args) {
|
||||
def task = new CompanyAddressCompare()
|
||||
task.run()
|
||||
}
|
||||
def run(){
|
||||
ExcelUtils.modify("D:\\数据\\公司及地址\\公司及地址.xlsx",{row->
|
||||
if(row['D'] && !row['I'] && row['B'].contains("宁波甬城农村商业银行")){
|
||||
def city = CommonUtils.parseCityInAddress(row["D"])
|
||||
if(!city){
|
||||
city = CommonUtils.parseCityInAddress(row["B"])
|
||||
}
|
||||
if(city){
|
||||
def similarData = null
|
||||
// if(isAddress(row["D"])){
|
||||
def simpleAddress = row["D"].toString().replace("中国(上海)自由贸易试验区","").replaceAll(/\\d+层/,"").replaceAll(/[一二三四五六]楼/,"")
|
||||
if(simpleAddress && simpleAddress.contains("号")){
|
||||
simpleAddress = simpleAddress.substring(0, simpleAddress.lastIndexOf("号")+1)
|
||||
}
|
||||
def pois = GaoDeUtils.search(simpleAddress, city, null)
|
||||
similarData = findSimilarDatas(pois, row["B"], simpleAddress, city, null)
|
||||
println row['C']+" "+pois.size()
|
||||
// }
|
||||
|
||||
if(similarData){
|
||||
row['F'] = similarData.address
|
||||
row['G'] = similarData.name
|
||||
row['H'] = similarData.score
|
||||
}
|
||||
}
|
||||
}
|
||||
}, 0)
|
||||
}
|
||||
|
||||
|
||||
def findSimilarDatas(pois, name, address, city, rMap){
|
||||
def data = null
|
||||
def maxScore = 0
|
||||
|
||||
for(def poi : pois){
|
||||
if(!poi.cityname){
|
||||
def m = getMapRegion(poi.location)
|
||||
if(m){
|
||||
poi.cityname = m.city
|
||||
poi.pname = m.province
|
||||
poi.township = m.township
|
||||
poi.adname = m.district
|
||||
}
|
||||
}
|
||||
if(poi.cityname.contains(city)){
|
||||
def searchName = poi.name
|
||||
def nameScore = CosineSimilarAlgorithm.getSimilarityRatio(searchName, name);
|
||||
def addressScore = 0
|
||||
if(poi.address){
|
||||
def poiAddress = poi.address.replace("号楼","号")
|
||||
addressScore = CosineSimilarAlgorithm.getSimilarityRatio(poiAddress, address);
|
||||
if(poi.address.contains("号") && address.contains("号") && (poiAddress.contains(address) || address.contains(poiAddress))){
|
||||
addressScore = addressScore + 0.5
|
||||
}
|
||||
}
|
||||
|
||||
def totalScore = nameScore + addressScore
|
||||
if(totalScore > maxScore){
|
||||
maxScore = totalScore
|
||||
data = poi
|
||||
data.score = totalScore
|
||||
}
|
||||
println searchName+" "+name+" "+totalScore + " "+ (poi.address?poi.address:"")
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
def isAddress(address){
|
||||
def REG_ADDR1 = "[\u4e00-\u9fa5]{2,20}[路|巷|街|弄|街道|大道|大街|巷子|公路|马路|大道西段]\\d+";
|
||||
def pattern = Pattern.compile(REG_ADDR1);
|
||||
def matcher = pattern.matcher(address)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
def getMapRegion(def location){
|
||||
def jsonObj = GaoDeApiUtils.geocode(location)
|
||||
def district = null
|
||||
if(jsonObj && jsonObj.status=="1"){
|
||||
def detailMap = jsonObj.regeocode.addressComponent
|
||||
if(!detailMap.city){
|
||||
if(detailMap.province && detailMap.province.toString().endsWith("市")){
|
||||
detailMap.city = detailMap.province
|
||||
}
|
||||
}
|
||||
return detailMap
|
||||
}
|
||||
return district;
|
||||
}
|
||||
|
||||
}
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
class CrawlDistrictIdSelect {
|
||||
def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol();
|
||||
|
||||
static DBCollection districtseedsColl = MongoUtils.getCollectionByAuth20190102("seeddb", "district_seeds")
|
||||
static DBCollection districtseeds2Coll = MongoUtils.getCollectionByAuth20190102("seeddb_dev", "district_seeds")
|
||||
|
||||
static main(args) {
|
||||
def task = new CrawlDistrictIdSelect()
|
||||
task.run()
|
||||
// task.synch()
|
||||
}
|
||||
|
||||
def synch(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = districtseeds2Coll.find(q)
|
||||
def list = []
|
||||
for(def d :dbcursor){
|
||||
list << d
|
||||
if(list.size()==10000){
|
||||
districtseedsColl.insert(list)
|
||||
list = []
|
||||
}
|
||||
}
|
||||
|
||||
if(list.size()>0){
|
||||
districtseedsColl.insert(list)
|
||||
}
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findCrawlData()
|
||||
datas.each {k,v->
|
||||
if(v){
|
||||
for(def d : v){
|
||||
d.c_date = new Date()
|
||||
d.m_date = new Date()
|
||||
districtseedsColl.save(d)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findCrawlData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city","福州")
|
||||
q.put("source", "安居客")
|
||||
q.put("s_date",new BasicDBObject('$gte', 202001))
|
||||
def dbcursor = thirdPartyColl.find(q)
|
||||
def retMap = [:]
|
||||
def prefix = "/community/view/"
|
||||
def keyList =[]
|
||||
for(def db : dbcursor){
|
||||
def url = null
|
||||
if(!db.url){
|
||||
if(db.district_id){
|
||||
url = db.district_id
|
||||
}
|
||||
}else{
|
||||
url = db.url
|
||||
}
|
||||
if(url && url.contains(prefix) ){
|
||||
def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.size())
|
||||
if(district_id=="0"){
|
||||
continue
|
||||
}
|
||||
def shortCity = url.substring(url.indexOf("https://")+8, url.indexOf(".anjuke.com"))
|
||||
def region = RegionData.formatRegion(db.city, db.region)
|
||||
def key = shortCity +"#"+district_id
|
||||
|
||||
def ids = retMap.get(key)
|
||||
if(!ids){
|
||||
ids = []
|
||||
retMap.put(key, ids)
|
||||
}
|
||||
|
||||
if(!keyList.contains(key)){
|
||||
def map = new BasicDBObject()
|
||||
map.root = "anjuke.com"
|
||||
map.city_short= shortCity
|
||||
map.city = db.city
|
||||
map.region = region
|
||||
map.district_id = district_id
|
||||
map.source_url = url
|
||||
ids << map
|
||||
|
||||
keyList << key
|
||||
}
|
||||
}else{
|
||||
println url +" 链接不合要求"
|
||||
}
|
||||
}
|
||||
return retMap
|
||||
}
|
||||
}
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.QueryOperators
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
class ExportToDealPrice {
|
||||
|
||||
DBCollection DEAL_PRICE_MATCH_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price_match", "192.168.0.43")
|
||||
DBCollection DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43")
|
||||
|
||||
DBCollection TRADE_MATCH_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "trade_match_price", "192.168.0.136")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50)
|
||||
final AtomicInteger count = new AtomicInteger()
|
||||
|
||||
static main(args) {
|
||||
def task = new ExportToDealPrice();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def datas = findDatas()
|
||||
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def data = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
count.addAndGet(1)
|
||||
def dbo = addPrice(data)
|
||||
if(dbo){
|
||||
data.weekend_avg_price = dbo.avg_price
|
||||
}
|
||||
}catch(Exception e){
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
save(datas);
|
||||
|
||||
}
|
||||
|
||||
def save(def datas){
|
||||
def ret = [];
|
||||
def index = 0;
|
||||
for(def dbo : datas){
|
||||
ret << dbo;
|
||||
index++;
|
||||
if(ret.size()%10000==0){
|
||||
println "µÚ "+(index/10000)+" ¸ö"+ret.size()
|
||||
TRADE_MATCH_PRICE_COLL.insert(ret);
|
||||
ret = [];
|
||||
}
|
||||
}
|
||||
if(ret.size()>0){
|
||||
TRADE_MATCH_PRICE_COLL.insert(ret);
|
||||
}
|
||||
}
|
||||
|
||||
def addPrice(data){
|
||||
def weekend = DateUtils.getFirstDayOfWeek(data.trade_date)
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", data.match_city)
|
||||
q.put("region", data.match_region)
|
||||
q.put("district_name", data.match_name)
|
||||
q.put("weekend", weekend)
|
||||
def dbo = DISTRICT_STAT_COLL.findOne(q)
|
||||
if(!dbo){
|
||||
q.put("weekend", new BasicDBObject(QueryOperators.LTE, weekend))
|
||||
dbo = DISTRICT_STAT_COLL.findOne(q)
|
||||
}
|
||||
println count.get()+" "+data.city +" "+data.match_name
|
||||
return dbo
|
||||
}
|
||||
|
||||
def findDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2020-04-01","yyyy-MM-dd")))
|
||||
def dbcursor = DEAL_PRICE_MATCH_COLL.find(q)
|
||||
def rets = []
|
||||
for(def db : dbcursor){
|
||||
rets << db
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
}
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.fangjia.script.util.DbUtils
|
||||
|
||||
class FZloationUpdate {
|
||||
|
||||
static main(args) {
|
||||
def task = new FZloationUpdate();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
def districts = CommonDatautils.findAllDistricts("福州", "district", ["city", "region","name","address","lat","lng"]);
|
||||
def index = 0
|
||||
def locationMap = getMoreSameLocationDist(districts)
|
||||
def total = 0
|
||||
locationMap.each {k,v->
|
||||
total = updateLocation(v, total);
|
||||
println "---------------------------------------------------------------------------------------------------------------"
|
||||
}
|
||||
}
|
||||
|
||||
def updateLocation(def lists, def total){
|
||||
|
||||
|
||||
for(def dist : lists){
|
||||
|
||||
def addrDistMap = getLocation(dist, dist.address)
|
||||
def nameDistMap = getLocation(dist, dist.region + dist.name)
|
||||
|
||||
if((addrDistMap && addrDistMap.distance>1000) && (nameDistMap && nameDistMap.distance>1000)){
|
||||
println dist.region +" "+dist.name+" "+dist.address+" addrDist:"+(addrDistMap!=null?addrDistMap.distance:"")+" addrKey:"+(addrDistMap!=null?addrDistMap.keyword:"")+" nameDist:"+(nameDistMap!=null?nameDistMap.distance:"")+" nameKey:"+(nameDistMap!=null?nameDistMap.keyword:"")
|
||||
// println "[${index}]"+ dist.region+" "+dist.name+" "+dist.address+" addrDist:"+addrDistMap.distance+"(${addrDistMap.level}-${addrDistMap.location}-${addrDistMap.keyword}) nameDist:"+nameDistMap.distance+"(${nameDistMap.level}-${nameDistMap.location}-${nameDistMap.keyword})"
|
||||
|
||||
if(addrDistMap){
|
||||
def addrKey = deleteRegion(addrDistMap.keyword, dist.region).replace("|","");
|
||||
def nameKey = deleteRegion(nameDistMap.keyword, dist.region);
|
||||
|
||||
def baidu = null
|
||||
def lat = null
|
||||
def lng = null
|
||||
def upldateFlag = ""
|
||||
if(addrDistMap && addrDistMap.region.contains(dist.region) && (addrKey.contains("路") || addrKey.contains("大道")) && addrKey.contains("号") && dist.addr == addrKey){
|
||||
lat = addrDistMap.lat
|
||||
lng = addrDistMap.lng
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Address"
|
||||
}else if(nameDistMap && nameDistMap.region.contains(dist.region) && nameKey == dist.name){
|
||||
lng = nameDistMap.lng
|
||||
lat = nameDistMap.lat
|
||||
baidu = BaiDuApiUtils.geoconv(lng, lat, 3)
|
||||
upldateFlag = "Name"
|
||||
}
|
||||
if(baidu){
|
||||
total++
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("m_date", new Date())
|
||||
Long baiduLat = baidu.y*10000000000L
|
||||
Long baiduLng = baidu.x*10000000000L
|
||||
udata.put("lat2", baiduLat)
|
||||
udata.put("lng2", baiduLng)
|
||||
Long lat0 = lat*10000000000L
|
||||
Long lng0 = lng*10000000000L
|
||||
udata.put("lat", lat0)
|
||||
udata.put("lng", lng0 )
|
||||
DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata))
|
||||
println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
def getMoreSameLocationDist(def districts){
|
||||
def locationMap = [:]
|
||||
for(def i=0;i<districts.size();i++){
|
||||
def dist = districts[i]
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def locationKey = lng + "-" + lat
|
||||
def list = locationMap.get(locationKey)
|
||||
if(!list){
|
||||
list = []
|
||||
locationMap.put(locationKey, list);
|
||||
}
|
||||
list << dist
|
||||
}
|
||||
def resMap = [:]
|
||||
locationMap.each{k,v->
|
||||
if(v.size()>=2){
|
||||
|
||||
v.sort{ a,b->
|
||||
return a.name.size().compareTo(b.name.size())
|
||||
}
|
||||
resMap.put(k,v)
|
||||
}
|
||||
}
|
||||
|
||||
def lastRetMaps = [:]
|
||||
resMap.each{k,v->
|
||||
def swname = null
|
||||
def count = 0
|
||||
for(def i=0;i<v.size();i++){
|
||||
def d = v[i]
|
||||
if(i==0){
|
||||
if(d.name.contains("期")||d.name.contains("区")){
|
||||
swname = d.name.replace("一期","").replace("二期","").replace("三期","").replace("四期","").replace("五期","").replace("六期","").replace("一区","").replace("二区","").replace("三区","").replace("四区","").replace("五区","").replace("A区","").replace("B区","").replace("C区","").replace("D区","")
|
||||
}else{
|
||||
swname = d.name
|
||||
}
|
||||
}else{
|
||||
if(d.name.contains(swname)||swname.contains(d.name)){
|
||||
count++
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if(count+1!=v.size()){
|
||||
lastRetMaps.put(k,v)
|
||||
}
|
||||
}
|
||||
return lastRetMaps;
|
||||
}
|
||||
|
||||
|
||||
def deleteRegion(def s, def region){
|
||||
def key = ""
|
||||
if(s.indexOf(region)>-1){
|
||||
if(s.indexOf(region+"区")>-1){
|
||||
key = s.substring(s.indexOf(region+"区")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"县")>-1){
|
||||
key = s.substring(s.indexOf(region+"县")+region.size()+1, s.size());
|
||||
}else if(s.indexOf(region+"市")>-1){
|
||||
key = s.substring(s.indexOf(region+"市")+region.size()+1, s.size());
|
||||
}
|
||||
}else{
|
||||
key = s;
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
|
||||
|
||||
def getLocation(dist, addr){
|
||||
|
||||
if(addr){
|
||||
def rMap = [:]
|
||||
def reps = GaoDeApiUtils.geo(dist.city, addr);
|
||||
if(reps && reps.count.toInteger()>=1){
|
||||
def map = reps.geocodes.get(0)
|
||||
def location = map.location
|
||||
def level = map.level
|
||||
if(location){
|
||||
def locMap = location.split(",")
|
||||
def lat = dist.lat/10000000000.0;
|
||||
def lng = dist.lng/10000000000.0;
|
||||
def distance = GeoHelper.GetDistance(locMap[1].toDouble(), locMap[0].toDouble(), lat, lng)
|
||||
rMap.distance = distance
|
||||
rMap.level = level
|
||||
rMap.lat = locMap[1].toDouble()
|
||||
rMap.lng = locMap[0].toDouble()
|
||||
rMap.keyword = map.formatted_address
|
||||
rMap.location = location
|
||||
rMap.region = map.district
|
||||
return rMap;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
+4
@@ -0,0 +1,4 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
class FzLocationBatch {
|
||||
}
|
||||
+20
-1
@@ -30,7 +30,9 @@ class JiangShuBlankDataStat {
|
||||
// task.matchHouse();
|
||||
|
||||
// task.villa()
|
||||
task.addressParse()
|
||||
// task.addressParse()
|
||||
|
||||
task.deleteProcess()
|
||||
}
|
||||
|
||||
def repeat(){
|
||||
@@ -288,5 +290,22 @@ class JiangShuBlankDataStat {
|
||||
return floor;
|
||||
}
|
||||
|
||||
def deleteProcess(){
|
||||
|
||||
def datas = ExcelUtils.read("D:\\数据\\江苏农行\\江苏2千.xlsx",0,1)
|
||||
def addrList = []
|
||||
for(def d : datas){
|
||||
def addr = d['G']
|
||||
addrList << addr
|
||||
}
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\江苏农行\\200平以下.xlsx", { row ->
|
||||
def addr = row['C']
|
||||
if(addrList.contains(addr)){
|
||||
row['R'] = "已调查"
|
||||
}
|
||||
|
||||
},0);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
package com.fangjia.script.gj.test
|
||||
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
|
||||
Date date = new Date(1617211269868);
|
||||
print dateFormat.format(date);
|
||||
+1
-2
@@ -18,8 +18,7 @@ import org.apache.commons.lang.StringUtils
|
||||
class AutocompleteDataUpdaterBatch extends BaseBatch {
|
||||
|
||||
|
||||
|
||||
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "192.168.0.51", 29009)
|
||||
private static final DBCollection AC_COLL = MongoUtils.getCollectionByName("fangjia", "autocomplete", "mongo.db.fangjia.com")
|
||||
private static final DSCollection SEAWEED_COLL = new DataStore("seaweed.db.fangjia.com", 27007).getCollection("fangjia.seaweed")
|
||||
private static final DSCollection AUTOCOMPLETE_COLL = new DataStore("192.168.0.38", 27008).getCollection("fangjia.autocomplete")
|
||||
private static final long C_DATE = DateUtil.formatToYYYYMMDDhhmmss(new Date()).toLong()
|
||||
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
/**
|
||||
* 抓取总量监控
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/12
|
||||
*/
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SeawaterMonitor extends BaseBatch {
|
||||
|
||||
static final Long expectNumber = 400000;
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
// 重点城市
|
||||
static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州",
|
||||
"沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州",
|
||||
"南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州",
|
||||
"厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳",
|
||||
"武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德",
|
||||
"海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海",
|
||||
"西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理",
|
||||
"东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴",
|
||||
"淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远",
|
||||
"马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照",
|
||||
"拉萨","大同","文昌","陵水","保亭"];
|
||||
// 重点网站
|
||||
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SeawaterMonitor(), args)
|
||||
}
|
||||
|
||||
/**
|
||||
* 监控前一日抓取量
|
||||
* @return
|
||||
*/
|
||||
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
|
||||
def total
|
||||
String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
if (total < expectNumber) {
|
||||
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点城市近半月抓取量
|
||||
* @return
|
||||
*/
|
||||
static boolean countCityCrawlNumber(def startDate, def endDate) {
|
||||
def result = []
|
||||
result << ['城市', '抓取量']
|
||||
// 统计近半个月各重点城市抓取量
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
result << [data.city, data.total]
|
||||
})
|
||||
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计前一日重点城市抓取量,昨天为零则警报
|
||||
* @return
|
||||
*/
|
||||
static def monitorCityCrawlNumber(def startDate, def endDate) {
|
||||
def msg = "【重点城市抓取异常】"
|
||||
def set = new HashSet()
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.city)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : city) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点网站抓取情况,近一周为0则警报
|
||||
*/
|
||||
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
|
||||
def msg = "【重点网站抓取异常】"
|
||||
def set = new HashSet()
|
||||
def siteFormatted = site.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.root)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : site) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 近一周抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
|
||||
def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1))
|
||||
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime();
|
||||
def endDate = DateUtil.truncateDay(new Date()).getTime()
|
||||
|
||||
def result = []
|
||||
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorCityCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
if (result.size() > 0) {
|
||||
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
|
||||
println(msg)
|
||||
DingDingMessageUtil.sendTextMessage(access_token, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -14,8 +14,8 @@ class TestCrawlTotalCount {
|
||||
def run(){
|
||||
def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房"]
|
||||
for(def source :sources){
|
||||
def count1 = findTotalCount(source, 202101);
|
||||
def count2 = findTotalCount(source, 202102);
|
||||
def count1 = findTotalCount(source, 202102);
|
||||
def count2 = findTotalCount(source, 202103);
|
||||
println source +" "+count1 +" "+count2
|
||||
}
|
||||
}
|
||||
|
||||
+213
@@ -0,0 +1,213 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import net.sf.json.groovy.JsonSlurper
|
||||
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
|
||||
class AddFangSchoolRegion {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection third_price_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "third_party_price")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school2")
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
|
||||
|
||||
static main(args) {
|
||||
def task = new AddFangSchoolRegion();
|
||||
//task.process()
|
||||
//task.match()
|
||||
task.addMapLocation()
|
||||
}
|
||||
|
||||
def addMapLocation(){
|
||||
def q = new BasicDBObject()
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def index = 0
|
||||
for(def d : dbcursor){
|
||||
def data = findSchool(d.city, d.region, d.school_name)
|
||||
if(data && data.location){
|
||||
def location = data.location
|
||||
if(location){
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("school_map_lng", location[0])
|
||||
udata.put("school_map_lat", location[1])
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata))
|
||||
}
|
||||
|
||||
}
|
||||
index ++
|
||||
println index
|
||||
}
|
||||
}
|
||||
|
||||
def match(){
|
||||
def datas = findAllDatas()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
for(def i=0;i<datas.size;i++){
|
||||
def data = datas[i]
|
||||
fixedThreadPool.execute(
|
||||
new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
// if(!data.match_name){
|
||||
println i +" "+data.district_name
|
||||
matching(data);
|
||||
// }
|
||||
|
||||
} catch (Exception e) {
|
||||
println "Òì³£"
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
|
||||
|
||||
|
||||
def matching(def it){
|
||||
def rMap = matchDistrict(it.city, it.region, it.district_name)
|
||||
if(!rMap){
|
||||
rMap = matchDistrict(it.city, it.region, it.address)
|
||||
}
|
||||
|
||||
if(rMap){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("match_city", rMap.city)
|
||||
data.put("match_region", rMap.region)
|
||||
data.put("match_name", rMap.name)
|
||||
data.put("match_address", rMap.address)
|
||||
data.put("match_lat", rMap.lat)
|
||||
data.put("match_lng", rMap.lng)
|
||||
district_school_col.update(new BasicDBObject('_id', it.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def matchDistrict(def city, def region, def address){
|
||||
def url = "http://open.fangjia.com/address/matchV2?city="+city+"&address="+address+"&preciseLevel=strict&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB&category=district"
|
||||
String content = new HttpRequest().get(url, "utf-8");
|
||||
def rMap = [:]
|
||||
if(content){
|
||||
def doc = new JsonSlurper().parseText(content)
|
||||
if(doc && doc.code ==200 && doc.result){
|
||||
def dto = doc.result
|
||||
if(dto && dto.searchAddress){
|
||||
def ret = dto.matchResult
|
||||
if(ret.match){
|
||||
rMap.city = dto.city
|
||||
rMap.region = dto.district
|
||||
rMap.name = dto.name
|
||||
rMap.lat = dto.lat
|
||||
rMap.lng = dto.lng
|
||||
rMap.id = dto.id
|
||||
rMap.address = dto.address
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rMap;
|
||||
}
|
||||
|
||||
|
||||
def process(){
|
||||
def datas = findNoRegionData();
|
||||
findSimialData(datas);
|
||||
}
|
||||
|
||||
def findAllDatas(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("match_name", new BasicDBObject(QueryOperators.EXISTS, false))
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets =[]
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findNoRegionData(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("region", null)
|
||||
def dbcursor = district_school_col.find(q)
|
||||
def rets = []
|
||||
for(def d : dbcursor){
|
||||
rets << d
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findSimialData(datas){
|
||||
for(def d : datas){
|
||||
def obj = findThirdPriceData(d.city, d.url+"/")
|
||||
if(!obj){
|
||||
obj = findThirdPriceByName(d.city, d.district_name)
|
||||
}
|
||||
if(obj){
|
||||
d.region = obj.region
|
||||
if(!d.district_addrss){
|
||||
d.district_address = obj.address
|
||||
}
|
||||
updateDistrictSchool(d);
|
||||
|
||||
}else{
|
||||
println d.school_name+"" + d.district_name+"û²éµ½"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findThirdPriceData(def city, def url){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("district_id", url)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def findThirdPriceByName(def city, def name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("source","ËÑ·¿")
|
||||
q.put("city", city)
|
||||
q.put("name", name)
|
||||
def dbcursor = third_price_col.find(q).sort(new BasicDBObject("s_date", -1)).limit(1)
|
||||
def obj = null;
|
||||
for(def d : dbcursor){
|
||||
obj = d
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
def updateDistrictSchool(d){
|
||||
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("region", d.region)
|
||||
data.put("district_address", d.district_address)
|
||||
district_school_col.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', data))
|
||||
}
|
||||
|
||||
def findSchool(city, region, school_name){
|
||||
DBObject data = new BasicDBObject()
|
||||
data.put("city", city)
|
||||
data.put("region", region)
|
||||
data.put("name", school_name)
|
||||
return school_col.findOne(data)
|
||||
}
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import org.apache.commons.io.IOUtils
|
||||
import org.apache.http.HttpResponse
|
||||
import org.apache.http.client.HttpClient
|
||||
import org.apache.http.client.methods.HttpGet
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
|
||||
class AnjukeCrawl {
|
||||
|
||||
static main(args) {
|
||||
def task = new AnjukeCrawl();
|
||||
task.run()
|
||||
}
|
||||
|
||||
def run(){
|
||||
HttpClient client = HttpClients.createDefault();
|
||||
HttpGet get = new HttpGet("https://m.anjuke.com/cityList/");
|
||||
// get.setHeader("Upgrade-Insecure-Requests", "1");
|
||||
// get.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36");
|
||||
|
||||
// get.setHeader("Host","m.anjuke.com");
|
||||
// get.setHeader("Connection","keep-alive");
|
||||
// get.setHeader("Cache-Control","max-age=0");
|
||||
// get.setHeader('sec-ch-ua','"Chromium";v="88", "Google Chrome";v="88", ";Not A Brand";v="99"');
|
||||
// get.setHeader("sec-ch-ua-mobile","'?0");
|
||||
// get.setHeader("Upgrade-Insecure-Requests","1");
|
||||
def header = [:]
|
||||
header.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36");
|
||||
header.put("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
|
||||
// get.setHeader("Sec-Fetch-Site","none");
|
||||
// get.setHeader("Sec-Fetch-Mode","navigate");
|
||||
// get.setHeader("Sec-Fetch-User","?1");
|
||||
// get.setHeader("Sec-Fetch-Dest","document");
|
||||
//// get.setHeader("Accept-Encoding"," gzip, deflate, br");
|
||||
// get.setHeader("Accept-Language","zh-CN,zh;q=0.9");
|
||||
|
||||
|
||||
def s1 = ABuYunUtil.doGet("https://m.anjuke.com/cityList/", null, header, "utf-8")
|
||||
printl s1
|
||||
|
||||
HttpResponse response = client.execute(get);
|
||||
def s = getResponseBodyAsString(response)
|
||||
println s
|
||||
}
|
||||
|
||||
private static String getResponseBodyAsString(HttpResponse response) throws IOException {
|
||||
return IOUtils.toString(response.getEntity().getContent(), "UTF-8");
|
||||
}
|
||||
|
||||
}
|
||||
+344
@@ -0,0 +1,344 @@
|
||||
package com.fangjia.script.thirdPriceCrawl.other
|
||||
|
||||
import com.alibaba.fastjson.annotation.JSONField
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.util.ABuYunUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.httpclient.NameValuePair
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.http.HttpEntity
|
||||
import org.apache.http.client.entity.UrlEncodedFormEntity
|
||||
import org.apache.http.client.methods.CloseableHttpResponse
|
||||
import org.apache.http.client.methods.HttpPost
|
||||
import org.apache.http.impl.client.CloseableHttpClient
|
||||
import org.apache.http.impl.client.HttpClients
|
||||
import org.apache.http.message.BasicNameValuePair
|
||||
import org.apache.http.util.EntityUtils
|
||||
|
||||
import java.nio.charset.Charset
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
class GushuwangCrawl {
|
||||
|
||||
static DBCollection district_school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "district_school")
|
||||
static DBCollection school_col = MongoUtils.getCollectionByAuth20190102("fangjia_craw", "school")
|
||||
static main(args) {
|
||||
def task = new GushuwangCrawl();
|
||||
task.process()
|
||||
}
|
||||
|
||||
|
||||
void process(Object args) {
|
||||
|
||||
// chaxuequCrawl()
|
||||
suzhouCrawl();
|
||||
|
||||
}
|
||||
|
||||
def suzhouCrawl(){
|
||||
def page = 1
|
||||
def stop = false
|
||||
def list = []
|
||||
while(!stop){
|
||||
def s = schoolList(page);
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.ok){
|
||||
def datas = jsonObject.extra.list
|
||||
if(datas){
|
||||
list.addAll(datas)
|
||||
page++
|
||||
println page
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}else{
|
||||
stop = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(list){
|
||||
def schools = []
|
||||
for(def d : list){
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = "苏州"
|
||||
schoolData.region = RegionData.parseRegionFormAddress(schoolData.city , d.adr)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(d.schoolname)
|
||||
schoolData.name = d.schoolname
|
||||
schoolData.address = d.adr
|
||||
/**来源网站*/
|
||||
schoolData.source = "苏州学区查询"
|
||||
/**房源详情url*/
|
||||
schools << schoolData
|
||||
}
|
||||
|
||||
saveChaxuequSchool(schools)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
def chaxuequCrawl(){
|
||||
|
||||
def page = 1
|
||||
def stop = false
|
||||
|
||||
def schools = []
|
||||
|
||||
while(!stop){
|
||||
def s = chaxuequInfoCrawl("https://51yongka.com/fapi/school/list", "schoolName=&city=%E8%8B%8F%E5%B7%9E&area=&schoolNature=&schoolType=&page="+page+"&size=10")
|
||||
println "内容:"+s
|
||||
if(s){
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject.code == 200){
|
||||
def data = jsonObject.data
|
||||
def totalPage = data.pages
|
||||
if(data.list){
|
||||
page++
|
||||
if(page+1>totalPage){
|
||||
stop = true;
|
||||
}
|
||||
schools.addAll(data.list)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
print(schools.size());
|
||||
|
||||
def districtSchools = []
|
||||
def schoolInfos = []
|
||||
if(schools.size()>0){
|
||||
|
||||
def url = "https://51yongka.com/fapi/school/details"
|
||||
for(def d : schools){
|
||||
def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM"
|
||||
def s = chaxuequInfoCrawl(url, params)
|
||||
JSONObject jsonObject = JSONObject.fromObject(s);
|
||||
if(jsonObject && jsonObject.code==200){
|
||||
|
||||
if(jsonObject.data && jsonObject.data.nearbyPropertyList){
|
||||
def districts = jsonObject.data.nearbyPropertyList
|
||||
for(def dist : districts){
|
||||
def dataMap = new BasicDBObject()
|
||||
dataMap.city = jsonObject.data.city
|
||||
dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
dataMap.category = converCategory(jsonObject.data.schoolName)
|
||||
dataMap.district_name = dist.name.trim()
|
||||
dataMap.district_address = dist.address
|
||||
dataMap.school_name = jsonObject.data.schoolName.trim()
|
||||
dataMap.school_address = jsonObject.data.address
|
||||
dataMap.source = "查学区房"
|
||||
dataMap.url = url + "?"+params
|
||||
dataMap.telephone = jsonObject.data.officialTelephone;
|
||||
districtSchools << dataMap
|
||||
}
|
||||
}else{
|
||||
|
||||
def schoolData = new BasicDBObject()
|
||||
schoolData.province = "江苏"
|
||||
schoolData.city = jsonObject.data.city
|
||||
schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area)
|
||||
/**类型: 幼儿园,小学,初中*/
|
||||
schoolData.category = converCategory(jsonObject.data.schoolName)
|
||||
schoolData.name = jsonObject.data.schoolName.trim()
|
||||
schoolData.address = jsonObject.data.address
|
||||
/**来源网站*/
|
||||
schoolData.source = "查学区房"
|
||||
/**房源详情url*/
|
||||
schoolData.telephone = jsonObject.data.officialTelephone;
|
||||
schoolInfos << schoolData
|
||||
println jsonObject.data.schoolName +" 无周边楼盘"
|
||||
}
|
||||
|
||||
}
|
||||
println s;
|
||||
}
|
||||
}
|
||||
|
||||
println "小区:"+districtSchools.size()
|
||||
// saveChaxuequDistrictSchool(districtSchools)
|
||||
saveChaxuequSchool(schoolInfos)
|
||||
|
||||
}
|
||||
|
||||
def saveChaxuequSchool(schoolInfos){
|
||||
for(def ds : schoolInfos){
|
||||
def obj = findSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
def saveChaxuequDistrictSchool(def districtSchools){
|
||||
for(def ds : districtSchools){
|
||||
def obj = findDistrictSchoolData(ds.city, ds.school_name);
|
||||
if(!obj){
|
||||
district_school_col.save(ds)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def findSchoolData(city, school_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
return school_col.findOne(q)
|
||||
}
|
||||
|
||||
|
||||
def findDistrictSchoolData(city, school_name, district_name){
|
||||
def q = new BasicDBObject()
|
||||
q.put("city", city)
|
||||
q.put("school_name", school_name)
|
||||
q.put("district_name", district_name)
|
||||
return district_school_col.findOne(q)
|
||||
}
|
||||
|
||||
def converCategory(def schoolName){
|
||||
|
||||
if(schoolName){
|
||||
if(schoolName.contains("初中部")||schoolName.contains("中学")){
|
||||
return "初中"
|
||||
}else if(schoolName.contains("小学")){
|
||||
return "小学"
|
||||
}
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def static chaxuequInfoCrawl(url, bodyquery){
|
||||
def responseMap = [:]
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","51yongka.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","75")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html")
|
||||
headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
String s = doPost(url, bodyquery, headers)
|
||||
return s
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
|
||||
def static schoolList(page) {
|
||||
def responseMap = [:]
|
||||
String s = null
|
||||
try {
|
||||
//headers信息
|
||||
Map<String, String> headers = new LinkedHashMap<String, String>()
|
||||
headers.put("Host","m.gusuwang.com")
|
||||
headers.put("Connection","keep-alive")
|
||||
headers.put("Content-Length","40")
|
||||
headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat")
|
||||
headers.put("content-type","application/x-www-form-urlencoded")
|
||||
headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html")
|
||||
// headers.put("Accept-Encoding","gzip, deflate, br")
|
||||
|
||||
//params信息
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
params.put("secCode","198a53da6baa3aa6a5b99959a006be37");
|
||||
// //post请求
|
||||
def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool"
|
||||
s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers)
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
static String doPost(String strUrl, String content, Map headers) {
|
||||
String result = "";
|
||||
OutputStream outputStream = null;
|
||||
InputStream inputStream = null;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
try {
|
||||
URL url = new URL(strUrl);
|
||||
//通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection.
|
||||
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
|
||||
urlConnection.setRequestMethod("POST");
|
||||
//设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setConnectTimeout(30000);
|
||||
//设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常
|
||||
urlConnection.setReadTimeout(30000);
|
||||
//将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据
|
||||
urlConnection.setDoOutput(true);
|
||||
urlConnection.setDoInput(true);
|
||||
//设置通用请求属性为默认浏览器编码类型
|
||||
// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded");
|
||||
|
||||
|
||||
if (headers != null && headers.keySet().size() > 0) {
|
||||
for (String head : headers.keySet()) {
|
||||
urlConnection.setRequestProperty(head, headers.get(head));
|
||||
}
|
||||
}
|
||||
//getOutputStream()返回的输出流,用于写入参数数据。
|
||||
outputStream = urlConnection.getOutputStream();
|
||||
outputStream.write(content.getBytes());
|
||||
outputStream.flush();
|
||||
outputStream.close();
|
||||
//此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。
|
||||
inputStream = urlConnection.getInputStream();
|
||||
|
||||
|
||||
// byte[] data = new byte[1024];
|
||||
// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1
|
||||
// while (inputStream.read(data) != -1) {
|
||||
// String s = new String(data, Charset.forName("utf-8"));
|
||||
// sb.append(s);
|
||||
// }
|
||||
|
||||
InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8");
|
||||
char[] buff = new char[1024];
|
||||
int length = 0;
|
||||
while ((length = reader.read(buff)) != -1) {
|
||||
String s = new String(buff, 0, length);
|
||||
sb.append(s);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}finally{
|
||||
if(outputStream!=null){
|
||||
outputStream.close();
|
||||
}
|
||||
if(inputStream!=null){
|
||||
inputStream.close();
|
||||
}
|
||||
}
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
@@ -425,6 +425,7 @@ class ABuYunUtil {
|
||||
buffer.append(bf, 0, len);
|
||||
}
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace()
|
||||
throw new Exception("doGet(String targetUrl, Map<String, String> params, Map<String, String> headers, String charset)异常, 异常信息为:" + e);
|
||||
} finally {
|
||||
// println("本次请求响应的connection.getResponseCode()为:" + connection.getResponseCode())
|
||||
|
||||
@@ -44,7 +44,7 @@ class ElasticUtils {
|
||||
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
@@ -52,7 +52,7 @@ class ElasticUtils {
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.206"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
@@ -140,13 +140,12 @@ class ElasticUtils {
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import com.mongodb.DBCursor
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
|
||||
class HouseUtils{
|
||||
static main(args){
|
||||
|
||||
+2
-2
@@ -121,9 +121,9 @@ public class OpenApiClient {
|
||||
params.put("suggestName", suggestName)
|
||||
}
|
||||
|
||||
String content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
|
||||
String content = new HttpRequest().get("http://open.fangjia.com/address/match", params, "utf-8")
|
||||
while (content == null) {
|
||||
content = new HttpRequest().get("http://192.168.10.199:9180/address/realMatch", params, "utf-8")
|
||||
content = new HttpRequest().get("http:///open.fangjia.com/address/match", params, "utf-8")
|
||||
println("${city} ${input} ${cat}程序调用异常")
|
||||
Thread.sleep(6000)
|
||||
}
|
||||
|
||||
+12
-2557
File diff suppressed because it is too large
Load Diff
@@ -6,34 +6,50 @@ import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
|
||||
def count = 0
|
||||
List<List<String>> data = Collections.synchronizedList([])
|
||||
data << ["ID","城市","区域","小区名","经度","纬度","街道"]
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Êý¾Ý.xlsx",{List item->
|
||||
// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){
|
||||
// item["D"] = "´æÔÚ"
|
||||
// }else{
|
||||
// item["D"] = "²»´æÔÚ"
|
||||
// }
|
||||
//})
|
||||
|
||||
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com")
|
||||
ProvinceData.getAllCity("¸£½¨").each {def city->
|
||||
int count = 0
|
||||
def s = new TreeMap()
|
||||
DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat->
|
||||
count ++
|
||||
int credit = (stat.credit * 100)/10
|
||||
s[credit] = s.get(credit, 0) + 1
|
||||
}
|
||||
|
||||
def data = []
|
||||
data << city
|
||||
data << count
|
||||
for(int i=0; i <=10; i++){
|
||||
data << s.get(i, 0)
|
||||
}
|
||||
println(data.join(","))
|
||||
CommonUtils.findAllCity().each {def city->
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
if(StringUtils.isBlank(item.township) && item.lng){
|
||||
count++
|
||||
def lng = item.lng/10000000000d
|
||||
def lat = item.lat/10000000000d
|
||||
def location = "${lng},${lat}"
|
||||
JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
def row = []
|
||||
row << item._id.toString()
|
||||
row << item.city
|
||||
row << item.region
|
||||
row << item.name
|
||||
row << lng
|
||||
row << lat
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.id = item._id.toString()
|
||||
dbo.city = item.city
|
||||
dbo.region = item.region
|
||||
dbo.name = item.name
|
||||
dbo.lng = lng
|
||||
dbo.lat = lat
|
||||
if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
|
||||
row << resp.regeocode.addressComponent.township
|
||||
dbo.township = resp.regeocode.addressComponent.township
|
||||
}
|
||||
println(row)
|
||||
data << row
|
||||
SEAWEED_TOWNSHIP.insert(dbo)
|
||||
}
|
||||
})
|
||||
}
|
||||
ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
+28
-287
@@ -1,6 +1,8 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
@@ -9,294 +11,33 @@ import net.sf.json.JSONObject
|
||||
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
//
|
||||
//def map = [:]
|
||||
//ExcelUtils.read("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批0829.xlsx", 1).each {List item->
|
||||
// if(item["AF"] && item["AH"]){
|
||||
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
// map.get(key, []) << key
|
||||
// }
|
||||
//}
|
||||
|
||||
//ExcelUtils.modify("D:\\商务数据\\上海农商行\\测试数据(上海)-上海农商行第二批(房价网交付0905).xlsx", { List item ->
|
||||
//// if(item["AF"] && item["AH"]){
|
||||
//// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
//// List nums = map.get(key)
|
||||
//// item["AT"] = nums.size()
|
||||
//// }
|
||||
//
|
||||
// def city = "上海"
|
||||
// def addr = item["AC"]
|
||||
// if(addr){
|
||||
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
|
||||
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
// println searchAddress
|
||||
//// item["AD"] = searchAddress.province
|
||||
//// item["AE"] = searchAddress.city
|
||||
//// item["AF"] = searchAddress.district
|
||||
// if(searchAddress.standardAddress){
|
||||
//// item["AG"] = searchAddress.street
|
||||
//// if(searchAddress.streetNumber){
|
||||
//// item["AG"] = item["AG"] + searchAddress.streetNumber
|
||||
//// }
|
||||
//
|
||||
// if(searchAddress.township){
|
||||
// item["AP"] = searchAddress.township
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if(item["AG"]){
|
||||
// def addrMap = getRoad(item["AG"])
|
||||
// if(addrMap){
|
||||
// if(addrMap.road){
|
||||
// item["AQ"] = addrMap.road
|
||||
// }
|
||||
// if(addrMap.roadNumber){
|
||||
// item["AR"] = addrMap.roadNumber
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
//// if (item["AH"] && item["AF"]) {
|
||||
//// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["AE"], item["AF"], item["AH"]))
|
||||
//// if (seaweed) {
|
||||
//// if (item["AR"] && item["AS"]) {
|
||||
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2, item["AS"].toInteger(), item["AR"].toInteger())
|
||||
//// if (eva) {
|
||||
//// item["AL"] = eva.avgPrice
|
||||
//// item["AM"] = eva.totalPrice
|
||||
//// }
|
||||
//// } else {
|
||||
//// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["O"], seaweed.estate_type2)
|
||||
//// if (eva) {
|
||||
//// item["AL"] = eva.avgPrice
|
||||
//// item["AM"] = eva.totalPrice
|
||||
//// }
|
||||
//// }
|
||||
//// }
|
||||
////
|
||||
//// }
|
||||
//
|
||||
//},1)
|
||||
|
||||
|
||||
// if(item["AK"] =~ /^\d{4}$/){
|
||||
// item["AR"] = item["AK"].toString().substring(0, 2)
|
||||
// }else if(item["AK"] =~ /^\d{3}$/){
|
||||
// item["AR"] = item["AK"].toString().substring(0, 1)
|
||||
// }
|
||||
// if(item["AR"] && item["AH"]){
|
||||
// def key = "${item["AE"]}_${item["AF"]}_${item["AH"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AS"] = nums[nums.size() - 1]
|
||||
// item["AT"] = nums.unique().size()
|
||||
// }
|
||||
// println(item)
|
||||
// item["AG"] = null
|
||||
// def city = "上海"
|
||||
// def addr = item["AC"]
|
||||
// def area = item["o"]
|
||||
// JSONObject resp = OpenApiClient.addressMatch(city, addr, "property")
|
||||
// JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
// item["AD"] = searchAddress.province
|
||||
// item["AE"] = searchAddress.city
|
||||
// item["AF"] = searchAddress.district
|
||||
// if(searchAddress.standardAddress){
|
||||
// item["AG"] = searchAddress.street
|
||||
// if(searchAddress.streetNumber){
|
||||
// item["AG"] = item["AG"] + searchAddress.streetNumber
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// if(resp.getDouble("credit") > 0){
|
||||
// item["AE"] = resp.city
|
||||
// item["AF"] = resp.district
|
||||
// item["AH"] = resp.name
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
|
||||
// item["AL"] = eva.avgPrice
|
||||
// item["AM"] = eva.totalPrice
|
||||
// item["AN"] = new Date().format("yyyy-MM-dd")
|
||||
// if(!item["AG"]){
|
||||
// item["AG"] = resp.address
|
||||
// }
|
||||
// }
|
||||
// item["AI"] = searchAddress.buildingNumber
|
||||
// item["AJ"] = searchAddress.unitNumber
|
||||
// item["AK"] = searchAddress.roomNumber
|
||||
// println(item)
|
||||
//},1)
|
||||
//
|
||||
def sheetIndexs = ["湘潭": 0, "嘉兴": 1, "苏州": 2]
|
||||
sheetIndexs.each { String city, int sheetIndex ->
|
||||
// def map = [:]
|
||||
// ExcelUtils.read("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批0828(测试)1.xlsx", sheetIndex).each {List item->
|
||||
// if(item["Q"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// map.get(key, []) << key
|
||||
// }
|
||||
// }
|
||||
ExcelUtils.multiThreadingModify("D:\\商务数据\\上海农商行\\异地测试-上海农商行第二批(房价网交付0905).xlsx", { List item ->
|
||||
// if(item["Q"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AE"] = nums.size()
|
||||
// }
|
||||
|
||||
// item["W"] = null
|
||||
// item["X"] = null
|
||||
|
||||
// BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["P"], item["Q"], item["S"]))
|
||||
// if(seaweed){
|
||||
// if(item["AC"] && item["AD"]){
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2, item["AD"].toInteger(), item["AC"].toInteger())
|
||||
// if(eva){
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// }
|
||||
// }else{
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(seaweed.city, seaweed.region, seaweed.name, item["L"], seaweed.estate_type2)
|
||||
// if(eva){
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(item["AC"] && item["S"]){
|
||||
// def key = "${item["P"]}_${item["Q"]}_${item["S"]}".toString()
|
||||
// List nums = map.get(key)
|
||||
// nums.sort()
|
||||
// item["AD"] = nums[nums.size() - 1]
|
||||
// item["AE"] = nums.unique().size()
|
||||
// }
|
||||
|
||||
// if(item["V"] =~ /^\d{4}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 2)
|
||||
// }else if(item["V"] =~ /^\d{3}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 1)
|
||||
// }else if(item["V"] =~ /^\d{6,7}$/){
|
||||
// item["AC"] = item["V"].toString().substring(0, 2).replaceAll("^0", "")
|
||||
// }
|
||||
|
||||
|
||||
def addr = item["B"].toString().replaceAll("(", "").replace('区/县', "区").replace("街道/镇", "镇")
|
||||
.replaceAll(")", "").replaceAll("【.*】", "").replaceAll("室/单元", "号")
|
||||
// def area = item["L"]
|
||||
|
||||
|
||||
def region = null
|
||||
if (city == "嘉兴") {
|
||||
region = "嘉善"
|
||||
} else if (city == "苏州") {
|
||||
region = "昆山"
|
||||
}
|
||||
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property", false, region)
|
||||
JSONObject searchAddress = resp.getJSONObject("searchAddress")
|
||||
|
||||
def township = ""
|
||||
if(searchAddress && searchAddress.township){
|
||||
item["AP"] = searchAddress.township
|
||||
township = searchAddress.township
|
||||
}
|
||||
if(item["S"]){
|
||||
def addrMap = getRoad(item["S"])
|
||||
if(addrMap){
|
||||
if(addrMap.road){
|
||||
def s = addrMap.road;
|
||||
if(township){
|
||||
if(township.contains(s)){
|
||||
item["AQ"] = s.replace(township, "")
|
||||
}else{
|
||||
if(township.contains("镇")){
|
||||
township = township.replace("镇","")
|
||||
}
|
||||
item["AQ"] = s.replace(township, "")
|
||||
}
|
||||
}else{
|
||||
item["AQ"] = addrMap.road
|
||||
}
|
||||
}
|
||||
if(addrMap.roadNumber){
|
||||
item["AR"] = addrMap.roadNumber
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
println searchAddress
|
||||
// item["O"] = searchAddress.province
|
||||
// item["P"] = searchAddress.city
|
||||
// item["Q"] = searchAddress.district
|
||||
// if (searchAddress.standardAddress) {
|
||||
// item["R"] = searchAddress.street
|
||||
// if (searchAddress.streetNumber) {
|
||||
// item["R"] = item["R"] + searchAddress.streetNumber
|
||||
// }
|
||||
// }
|
||||
// if (resp.getDouble("credit") > 0) {
|
||||
// item["O"] = resp.province
|
||||
// item["P"] = resp.city
|
||||
// item["Q"] = resp.district
|
||||
// item["S"] = resp.name
|
||||
// JSONObject eva = OpenApiClient.evaluateDistrictPrice2(resp.city, resp.district, resp.name, area, resp.propertyType)
|
||||
// item["W"] = eva.avgPrice
|
||||
// item["X"] = eva.totalPrice
|
||||
// item["Y"] = new Date().format("yyyy-MM-dd")
|
||||
// if (!item["R"]) {
|
||||
// item["R"] = resp.address
|
||||
// }
|
||||
// }
|
||||
// item["T"] = searchAddress.buildingNumber
|
||||
// item["U"] = searchAddress.unitNumber
|
||||
// item["V"] = searchAddress.roomNumber
|
||||
// if(city == "湘潭"){
|
||||
// String roomNumber = DataUtil.match(addr, Pattern.compile("\\d+(号|(号房))\$"))
|
||||
// if(roomNumber){
|
||||
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
|
||||
// }
|
||||
// roomNumber = DataUtil.match(addr, Pattern.compile("\\d{6,8}\$"))
|
||||
// if(roomNumber){
|
||||
// item["V"] = roomNumber.replaceAll("号", '').replaceAll('房', "")
|
||||
// }
|
||||
//
|
||||
// }
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
|
||||
new File("D:\\Êý¾Ý\\ѧУÊý¾Ý").eachFile {File file->
|
||||
ExcelUtils.read(file.getAbsolutePath(), 0).each {List item->
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.city = item["A"]
|
||||
dbo.region = item["B"]
|
||||
dbo.name = item["C"]
|
||||
dbo.address = item["E"]
|
||||
dbo.attribute = item["G"]
|
||||
dbo.property = item["H"]
|
||||
dbo.category = "Сѧ"
|
||||
dbo.location = [DataUtil.roundDouble(item["I"], 6), DataUtil.roundDouble(item["J"], 6)]
|
||||
school.insert(dbo)
|
||||
println(dbo)
|
||||
}
|
||||
ExcelUtils.read(file.getAbsolutePath(), 1).each {List item->
|
||||
BasicDBObject dbo = new BasicDBObject()
|
||||
dbo.city = item["A"]
|
||||
dbo.region = item["B"]
|
||||
dbo.name = item["C"]
|
||||
dbo.address = item["E"]
|
||||
dbo.attribute = item["G"]
|
||||
dbo.property = item["H"]
|
||||
dbo.category = "ÖÐѧ"
|
||||
dbo.location = [item["I"], item["J"]]
|
||||
school.save(dbo)
|
||||
println(item)
|
||||
}, sheetIndex)
|
||||
}
|
||||
|
||||
|
||||
def getRoad(def s){
|
||||
def rex = ".*(路|街道|街|大道|巷|道|里).*\\d+(弄|号)\$"
|
||||
Pattern pattern = Pattern.compile(rex);
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
List<String> list = new ArrayList();
|
||||
while (matcher.find()) {
|
||||
list.add(matcher.group());
|
||||
}
|
||||
if(list && list.size()>0){
|
||||
def s1 = list[0]
|
||||
def road = getRexText(s1, ".*(路|街道|街|大道|巷|道|里)")
|
||||
def num = getRexText(s1, "\\d+")
|
||||
return ["road":road, "roadNumber":num]
|
||||
}else{
|
||||
def road = getRexText(s, ".*(路|街道|街|大道|巷|道|里)")
|
||||
return ["road":road]
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def getRexText(s, regx){
|
||||
Pattern pattern = Pattern.compile(regx);
|
||||
Matcher matcher = pattern.matcher(s);
|
||||
|
||||
while (matcher.find()) {
|
||||
return matcher.group();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -1,9 +1,16 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.openqa.selenium.json.Json
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
|
||||
@@ -30,8 +37,6 @@ import org.apache.commons.lang3.StringUtils
|
||||
//}, 0,"结果")
|
||||
|
||||
|
||||
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\阳光保险重跑数据(1).xlsx", { List item ->
|
||||
// if (item["T"] == null || ![1, 2, 4].contains(item["T"].toString().toDouble().toInteger())) {
|
||||
// return
|
||||
@@ -114,43 +119,80 @@ import org.apache.commons.lang3.StringUtils
|
||||
// println("${key} ${names.join(",")}")
|
||||
// }
|
||||
//}
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\µØÖ·±ê×¼»¯.xlsx",{List item->
|
||||
String address = "";
|
||||
if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
|
||||
address = ProvinceData.getDisplay(item["B"])
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\地址标准化.xlsx",{List item->
|
||||
// String address = "";
|
||||
// if(StringUtils.isNotBlank(item["B"]) && !ProvinceData.isGovernmentCity(item["B"])){
|
||||
// address = ProvinceData.getDisplay(item["B"])
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["C"])){
|
||||
// String cityDisplay = CityData.getCityDisplay(item["C"]);
|
||||
// if(cityDisplay != null){
|
||||
// address = address + cityDisplay
|
||||
// }else{
|
||||
// address = address + item["C"] + "市"
|
||||
// }
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["D"])){
|
||||
// String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
|
||||
// if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
|
||||
// address = address + orgDistrict
|
||||
// }
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["E"])){
|
||||
// address = address + item["E"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["F"])){
|
||||
// address = address + item["F"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["G"])){
|
||||
// address = address + item["G"]
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["H"])){
|
||||
// address = address + item["H"] + "栋"
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["i"])){
|
||||
// address = address + item["i"] + "单元"
|
||||
// }
|
||||
// if(StringUtils.isNotBlank(item["J"])){
|
||||
// address = address + item["J"] + "室"
|
||||
// }
|
||||
// item["K"] = address
|
||||
// println(address)
|
||||
//})
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_MONGO_URI, "fangjia", "guiding_price")
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\深圳市住建局二手房指导价.xlsx", { List item ->
|
||||
if (item["G"]) {
|
||||
BasicDBObject seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["G"], item["H"], item["I"]))
|
||||
BasicDBObject guid = new BasicDBObject()
|
||||
guid.guiding_type = 0
|
||||
guid.estate_type = seaweed.estate_type2
|
||||
guid.house_type = 0
|
||||
guid.c_date = new Date()
|
||||
guid.guiding_person = "深圳房管局"
|
||||
guid.guiding_date = Date.parse('yyyyMMdd', '20210411')
|
||||
guid.category = "district"
|
||||
guid.guiding_value = (int)item["E"]
|
||||
guid.name = item["D"]
|
||||
guid.city = item["G"]
|
||||
guid.region = item["H"]
|
||||
guid.name = item["i"]
|
||||
school.insert(guid)
|
||||
println(guid)
|
||||
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["C"])){
|
||||
String cityDisplay = CityData.getCityDisplay(item["C"]);
|
||||
if(cityDisplay != null){
|
||||
address = address + cityDisplay
|
||||
}else{
|
||||
address = address + item["C"] + "ÊÐ"
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["D"])){
|
||||
String orgDistrict = RegionData.getOrgName(item["C"], item["D"]);
|
||||
if(StringUtils.isNotBlank(orgDistrict) && address.indexOf(orgDistrict) == -1){
|
||||
address = address + orgDistrict
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["E"])){
|
||||
address = address + item["E"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["F"])){
|
||||
address = address + item["F"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["G"])){
|
||||
address = address + item["G"]
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["H"])){
|
||||
address = address + item["H"] + "¶°"
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["i"])){
|
||||
address = address + item["i"] + "µ¥Ôª"
|
||||
}
|
||||
if(StringUtils.isNotBlank(item["J"])){
|
||||
address = address + item["J"] + "ÊÒ"
|
||||
}
|
||||
item["K"] = address
|
||||
println(address)
|
||||
// if(!item["F"]){
|
||||
// JSONObject resp = OpenApiClient.addressMatch("深圳", item["D"], 'property', false, item["B"])
|
||||
// if(resp.id){
|
||||
// BasicDBObject seaweed = DbUtils.seaweed.findById(resp.id)
|
||||
// if(seaweed){
|
||||
// item["F"] = seaweed._id.toString()
|
||||
// item["G"] = seaweed.city
|
||||
// item["H"] = seaweed.region
|
||||
// item["I"] = seaweed.name
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
//
|
||||
// }
|
||||
|
||||
})
|
||||
+140
-73
@@ -1,82 +1,149 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.util.HouseValidator
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import net.sf.json.JSONObject
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\商务数据\\建设银行\\副本5市成交案例(测试).xlsx",{List item->
|
||||
// if(item["R"]){
|
||||
// return
|
||||
// }
|
||||
// def calcDate = null
|
||||
// if(item["E"]){
|
||||
// calcDate = item["E"].toString().substring(0, 10)
|
||||
// }else{
|
||||
// calcDate = item["B"].toString().substring(0, 10)
|
||||
// }
|
||||
// def floor = item["I"].toString().toDouble().toInteger().toString()
|
||||
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
|
||||
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
|
||||
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
|
||||
// def params =[:]
|
||||
// params.city = item["F"]
|
||||
// params.district = item["p"]
|
||||
// params.name = item['Q']
|
||||
// params.size = item["C"].toString()
|
||||
// params.floor = floor
|
||||
// params.maxFloor = maxFloor
|
||||
// params.direction = dir
|
||||
// params.decoration = fit
|
||||
// params.calDate = calcDate
|
||||
// params.__refresh_cache = "1"
|
||||
// params.scope = "1"
|
||||
// JSONObject evaResp = OpenApiClient.evaluate(params)
|
||||
// if(evaResp.result && evaResp.result.credit > 0.4){
|
||||
// item["R"] = evaResp.result.totalPrice
|
||||
// }
|
||||
def school = MongoUtils.getDbCollection(Constants.FANGJIA_CRAWL_MONGO_URI, "fangjia_craw", "school2")
|
||||
//school.find(new BasicDBObject("source", "查学区")).each {BasicDBObject item->
|
||||
// item.category = item.attribute
|
||||
// item.attribute = null
|
||||
// school.save(item)
|
||||
// println(item)
|
||||
|
||||
// if(item["H"] != "中粮云景花园"){
|
||||
// return
|
||||
//}
|
||||
//ExcelUtils.read("D:\\Users\\wenyan\\Downloads\\无经纬度学校-结果.xlsx").each {List item->
|
||||
// if(item["P"] != null){
|
||||
// def location = item["P"].split(",") as List
|
||||
// def data = new BasicDBObject()
|
||||
// data.map_name = item["N"]
|
||||
// data.map_region = item["R"]
|
||||
// data.map_address = item["N"]
|
||||
// data.map_type = item["S"]
|
||||
// data.location = [location[0].toString().toDouble(),location[1].toString().toDouble()]
|
||||
// school.update(new BasicDBObject("_id", new ObjectId(item["A"])), new BasicDBObject('$set', data))
|
||||
// println(data)
|
||||
// }
|
||||
def city = item["F"]
|
||||
def region = item["G"]
|
||||
def name = item["H"].toString()
|
||||
def addr = item["D"]
|
||||
JSONObject resp = OpenApiClient.addressMatch(city, addr, "property" , false, region, name)
|
||||
if(resp.name){
|
||||
item["p"] = resp.district
|
||||
item['Q'] = resp.name
|
||||
//}
|
||||
regionCodeMap = [:]
|
||||
def cityCodeMap = ["苏州":"320500","合肥":"340100", "西安":"610000"]
|
||||
for(def city : cityCodeMap.keySet()){
|
||||
def datas = FangJiaAreaMapUtils.convertToOfficials(city);
|
||||
for(def d : datas){
|
||||
regionCodeMap.put(city+"-"+d.fangjia_district, d.district_code)
|
||||
}
|
||||
println(item)
|
||||
// def calcDate = null
|
||||
// if(item["E"]){
|
||||
// calcDate = item["E"].toString().substring(0, 10)
|
||||
// }else{
|
||||
// calcDate = item["B"].toString().substring(0, 10)
|
||||
// }
|
||||
// def floor = item["I"].toString().toDouble().toInteger().toString()
|
||||
// def maxFloor = item["J"].toString().toDouble().toInteger().toString()
|
||||
// def dir = HouseValidator.formatDirectionToCode(item["L"]).toString()
|
||||
// def fit = HouseValidator.formatFitmentToCode(item["M"]).toString()
|
||||
// def params =[:]
|
||||
// params.city = resp.city
|
||||
// params.district = resp.district
|
||||
// params.name = resp.name
|
||||
// params.size = item["C"].toString()
|
||||
// params.floor = floor
|
||||
// params.maxFloor = maxFloor
|
||||
// params.direction = dir
|
||||
// params.decoration = fit
|
||||
// params.calDate = calcDate
|
||||
// params.scope = "1"
|
||||
// JSONObject evaResp = OpenApiClient.evaluate(params)
|
||||
// if(evaResp.result && evaResp.result.credit > 0.4){
|
||||
// item["R"] = evaResp.result.totalPrice
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
}
|
||||
def data = Collections.synchronizedList([])
|
||||
data << ["ID","CITYID","DISTID","COMMNAME","LNG","LAT","ADDRESS","PS","PS_ADDR","PS_TYPE","PS_SEQNO","PS_LONGITUDE","PS_LATITUDE","MS","MS_ADDR","MS_TYPE","MS_SEQNO","MS_LONGITUDE","MS_LATITUDE","REMARK","CREATETIME","UPDATETIME","SOURCECODE"]
|
||||
cityCodeMap.keySet().each {def city->
|
||||
DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject seaweed->
|
||||
if(seaweed.lng){
|
||||
def item = []
|
||||
item << seaweed._id.toString()
|
||||
item << cityCodeMap.get(seaweed.city)
|
||||
item << regionCodeMap.get(seaweed.city + "-" + seaweed.region)
|
||||
item << seaweed.name
|
||||
def lng = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
|
||||
def lat = com.fangjia.core.util.DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
|
||||
item << lng
|
||||
item << lat
|
||||
item << seaweed.address
|
||||
BasicDBObject location = new BasicDBObject()
|
||||
BasicDBObject near = new BasicDBObject()
|
||||
BasicDBObject geometry = new BasicDBObject("type", "Point")
|
||||
geometry.append("coordinates", [lng, lat])
|
||||
near.append('$maxDistance', 1000)
|
||||
near.append('$geometry', geometry)
|
||||
location.append('$near', near)
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.append("location", location)
|
||||
query.append("attribute",new BasicDBObject('$in', ['省重点', '市重点', '区重点', '县重点', '省级示范']))
|
||||
query.append("category", new BasicDBObject('$in', ['中学']))
|
||||
BasicDBObject midSchool = school.findOne(query)
|
||||
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
|
||||
// query.append("location", location)
|
||||
// query.append("attribute", level)
|
||||
// midSchool = school.findOne(query)
|
||||
// if(midSchool){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(!midSchool){
|
||||
// query.removeField("attribute")
|
||||
// midSchool = school.findOne(query)
|
||||
// }
|
||||
BasicDBObject primarySchool=null
|
||||
query.append("category", new BasicDBObject('$in', ['小学']))
|
||||
primarySchool = school.findOne(query)
|
||||
// for(def level in ['省重点', '市重点', '区重点', '县重点', '普通']){
|
||||
// query.append("location", location)
|
||||
// query.append("attribute", level)
|
||||
// primarySchool = school.findOne(query)
|
||||
// if(primarySchool){
|
||||
// break
|
||||
// }
|
||||
// }
|
||||
// if(!primarySchool){
|
||||
// query.removeField("attribute")
|
||||
// primarySchool = school.findOne(query)
|
||||
// }
|
||||
if(primarySchool){
|
||||
item << primarySchool.name
|
||||
item << primarySchool.address
|
||||
item << primarySchool.attribute
|
||||
item << null
|
||||
item << primarySchool.location[0]
|
||||
item << primarySchool.location[1]
|
||||
}else {
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
}
|
||||
if(midSchool){
|
||||
item << midSchool.name
|
||||
item << midSchool.address
|
||||
item << midSchool.attribute
|
||||
item << null
|
||||
item << midSchool.location[0]
|
||||
item << midSchool.location[1]
|
||||
}else {
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
item << null
|
||||
}
|
||||
item << null
|
||||
Date c_date = seaweed.c_date
|
||||
Date m_date = seaweed.m_date
|
||||
if(!c_date && !m_date){
|
||||
c_date = DateUtil.add(new Date(), Calendar.YEAR, -1)
|
||||
}
|
||||
if(!c_date){
|
||||
c_date = m_date
|
||||
}
|
||||
if(!m_date){
|
||||
m_date = c_date
|
||||
}
|
||||
item <<c_date.format("yyyy-MM-dd HH:mm:ss")
|
||||
item <<m_date.format("yyyy-MM-dd HH:mm:ss")
|
||||
item << 'fjw'
|
||||
println(item)
|
||||
if(primarySchool || midSchool){
|
||||
data << item
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
ExcelUtils.write("D:\\无学区房数据.xlsx", data, "结果")
|
||||
|
||||
|
||||
})
|
||||
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
+23
-5
@@ -11,26 +11,44 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210501", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
def data = []
|
||||
def title = ["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg"]
|
||||
def param_keys = ["tid", "city", "district", "address", "name", "size", "propertyType"]
|
||||
param_keys.each {
|
||||
title << "参数_${it}"
|
||||
}
|
||||
def resMap_keys = ["city", "district", "name", "avgPrice", "totalPrice", "buildYear"]
|
||||
resMap_keys.each {
|
||||
title << "结果_${it}"
|
||||
}
|
||||
data << title
|
||||
def keys = new HashSet()
|
||||
while (startDate.before(endDate)) {
|
||||
calendar.add(Calendar.DATE, 1)
|
||||
def nextDate = calendar.getTime()
|
||||
count++
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
if(!result.buildYear || !result.avgPrice){
|
||||
if (!result.buildYear || !result.avgPrice) {
|
||||
return
|
||||
}
|
||||
def params = JSONObject.fromObject(doc.parameter_map)
|
||||
keys.addAll(params.keys())
|
||||
def key = "${params.address}_${params.size}".toString()
|
||||
if (!hashSet.contains(key)) {
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date.format("yyy"), doc.data_status, doc.code, doc.msg]
|
||||
param_keys.each {
|
||||
item << params[it]
|
||||
}
|
||||
resMap_keys.each {
|
||||
item << result[it]
|
||||
}
|
||||
data << item
|
||||
hashSet << key
|
||||
} else {
|
||||
|
||||
+47
-47
@@ -153,55 +153,55 @@ import org.apache.commons.lang3.StringUtils
|
||||
//}
|
||||
|
||||
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
|
||||
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
|
||||
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
|
||||
def coords = []
|
||||
def baidu_points = []
|
||||
def gaode_points = []
|
||||
for(int i=0; i < border.size() - 1; i=i+2){
|
||||
coords << "${border[i]},${border[i+1]}"
|
||||
if(coords.size() > 50){
|
||||
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
for(JSONObject it : json.result){
|
||||
baidu_points << "${it.x},${it.y}"
|
||||
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
}
|
||||
coords.clear()
|
||||
}
|
||||
}
|
||||
if(coords.size() > 0){
|
||||
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
for(JSONObject it : json.result){
|
||||
baidu_points << "${it.x},${it.y}"
|
||||
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
}
|
||||
}
|
||||
dbo.format_city = RegionData.formatCity(dbo.city)
|
||||
dbo.baidu_border = baidu_points.join(";")
|
||||
dbo.gaode_border = gaode_points.join(";")
|
||||
BAIDUMAP_BORDER_COLL.save(dbo)
|
||||
println(dbo)
|
||||
}
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
|
||||
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
|
||||
// if(seaweed){
|
||||
// item["A"] = seaweed._id.toString()
|
||||
// def names = [item['D']]
|
||||
// if(seaweed.alias){
|
||||
// names.addAll(seaweed.alias)
|
||||
//BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
|
||||
// def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
|
||||
// def coords = []
|
||||
// def baidu_points = []
|
||||
// def gaode_points = []
|
||||
// for(int i=0; i < border.size() - 1; i=i+2){
|
||||
// coords << "${border[i]},${border[i+1]}"
|
||||
// if(coords.size() > 50){
|
||||
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
// for(JSONObject it : json.result){
|
||||
// baidu_points << "${it.x},${it.y}"
|
||||
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
// }
|
||||
// coords.clear()
|
||||
// }
|
||||
// names.unique()
|
||||
// }
|
||||
// if(coords.size() > 0){
|
||||
// def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
// for(JSONObject it : json.result){
|
||||
// baidu_points << "${it.x},${it.y}"
|
||||
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
// gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
// }
|
||||
// }
|
||||
// dbo.format_city = RegionData.formatCity(dbo.city)
|
||||
// dbo.baidu_border = baidu_points.join(";")
|
||||
// dbo.gaode_border = gaode_points.join(";")
|
||||
// BAIDUMAP_BORDER_COLL.save(dbo)
|
||||
// println(dbo)
|
||||
//}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Ð¡ÇøÈºÏñ²âÊÔÊý¾Ý.xlsx",{List item->
|
||||
def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
|
||||
if(seaweed){
|
||||
// item["A"] = seaweed._id.toString()
|
||||
def names = [item['D']]
|
||||
if(seaweed.alias){
|
||||
names.addAll(seaweed.alias)
|
||||
}
|
||||
names.unique()
|
||||
// if(seaweed.lng){
|
||||
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
|
||||
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
|
||||
// }
|
||||
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
|
||||
// if(one){
|
||||
// item["I"] = one.baidu_border
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
|
||||
if(one){
|
||||
item["I"] = one.gaode_border
|
||||
}
|
||||
println(item)
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,46 @@
|
||||
package com.fangjia.script.wenyan.test
|
||||
|
||||
import com.aliyuncs.DefaultAcsClient
|
||||
import com.aliyuncs.IAcsClient
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse
|
||||
import com.aliyuncs.profile.DefaultProfile
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
|
||||
def city = item["A"]
|
||||
def address1 = item["B"]
|
||||
def address2 = item["C"]
|
||||
HttpRequest http = new HttpRequest()
|
||||
def params = new HashMap()
|
||||
params["city"] = city
|
||||
params["address_1"] = address1
|
||||
params["address_2"] = address2
|
||||
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
|
||||
JSONObject result = json.result
|
||||
item["D"] = result?.level
|
||||
item["E"] = result?.formattedAddress_1
|
||||
item["F"] = result?.formattedAddress_2
|
||||
item["G"] = result?.isSameAddress
|
||||
DefaultProfile profile = DefaultProfile.getProfile(
|
||||
"cn-hangzhou",
|
||||
"LTAIdaj4eSO79gzz",
|
||||
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
|
||||
IAcsClient client = new DefaultAcsClient(profile);
|
||||
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
|
||||
request.setServiceCode("addrp")
|
||||
request.setAppKey("Xcgtyhupey0h")
|
||||
request.setText("${address1};${address2}")
|
||||
GetAddressSimilarityResponse response = client.getAcsResponse(request)
|
||||
String data = response.getData()
|
||||
json = JSONObject.fromObject(data)
|
||||
item["H"] = json["sim_level"]["level"]
|
||||
item["I"] = json["sim_level"]["address_a"]
|
||||
item["J"] = json["sim_level"]["address_b"]
|
||||
item["K"] = json["sim_level"]["is_same"]
|
||||
println(json)
|
||||
})
|
||||
|
||||
+3
-3
@@ -7,14 +7,14 @@ import java.util.regex.Pattern;
|
||||
*/
|
||||
public interface AddressParserPatterns {
|
||||
|
||||
Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
|
||||
Pattern FULL_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([甲乙]?[零一二三四五六七八九十百千\\d]+甲?号?)(?!楼)[~\\-~、,-])?([甲乙]?[零一二三四五六七八九十百千\\d])+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
|
||||
Pattern STREET = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?");
|
||||
Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零|一三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
|
||||
Pattern STREET_NUMBER = Pattern.compile(".*([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)?[~\\--~、,])?([甲乙]?[零一二三四五六七八九十百千\\d]+甲?(号)(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
|
||||
Pattern STAGE_PATTERN = Pattern.compile("(([零一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([零一二三四五六七八九十百千\\d]+)期");
|
||||
Pattern PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|零|一|二|三|四|五|六|七|八|九|十)+(区|坊)");
|
||||
Pattern TOWN = Pattern.compile("^.+?(乡|镇|街道)+");
|
||||
Pattern NUMBER = Pattern.compile("\\d+");
|
||||
Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?)?)");
|
||||
Pattern STANDARD_ADDRESS = Pattern.compile("^.{2,}(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙]?([零一二三四五六七八九十百千\\d]))+号(?!楼)+([付附]+[零一二三四五六七八九十百千\\d]+(号)?+([-]+[零一二三四五六七八九十百千\\d]+(号)?)?)?)");
|
||||
Pattern DISTRICT = Pattern.compile("^.{2,}?(区|县)");
|
||||
|
||||
}
|
||||
|
||||
+225
-35
@@ -1,13 +1,12 @@
|
||||
package com.fangjia.service.address;
|
||||
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.GeoUtils;
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.common.DistrictKey;
|
||||
import com.fangjia.dal.common.ServerAreaData;
|
||||
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
|
||||
import com.fangjia.dal.mongo.model.AmapPoi;
|
||||
import com.fangjia.dal.mongo.model.bkoff.HouseLoudong;
|
||||
import com.fangjia.dal.mongo.model.bkoff.Loudong;
|
||||
import com.fangjia.dal.seaweed.dao.SeaweedDao;
|
||||
@@ -17,12 +16,14 @@ import com.fangjia.service.address.processor.AddressDistrictProcessors;
|
||||
import com.fangjia.service.address.processor.AddressProcessors;
|
||||
import com.fangjia.service.api.AliyunService;
|
||||
import com.fangjia.service.bo.AliyunBo;
|
||||
import com.fangjia.service.bo.MatchAddressScore;
|
||||
import com.fangjia.service.datastore.DistrictService;
|
||||
import com.fangjia.service.mongo.addrepo.AddressRepoService;
|
||||
import com.fangjia.service.mongo.bkoff.HouseLoudongService;
|
||||
import com.fangjia.service.mongo.bkoff.LoudongService;
|
||||
import com.google.common.base.Function;
|
||||
import com.google.common.collect.MapMaker;
|
||||
import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.log4j.Logger;
|
||||
import org.bson.types.ObjectId;
|
||||
@@ -45,7 +46,7 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
private LoudongService loudongService;
|
||||
private SeaweedDao seaweedDao;
|
||||
private AliyunService aliyunService;
|
||||
private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"};
|
||||
private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "township"};
|
||||
private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class);
|
||||
|
||||
|
||||
@@ -132,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
formatAddress(building);
|
||||
}
|
||||
return building;
|
||||
/*JSONObject result = getDataFromCRF("test");
|
||||
return new Building();*/
|
||||
}
|
||||
|
||||
@Override
|
||||
@@ -163,35 +166,188 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
return building;
|
||||
}
|
||||
Building matchedBuilding;
|
||||
//调用CRF算法切分数据
|
||||
/*JSONObject jsonObj = null;
|
||||
if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
|
||||
jsonObj = getDataFromCRF(addr.getOrgAddress());
|
||||
if(jsonObj != null){
|
||||
String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") +
|
||||
(jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") +
|
||||
(jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : "");
|
||||
if(StringUtils.isNotBlank(poi)){
|
||||
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
|
||||
addr.setKeywords(poi);
|
||||
}
|
||||
}
|
||||
}
|
||||
}*/
|
||||
//keywords使用aliyun接口返回的poi
|
||||
AliyunBo aliyunBo = null;
|
||||
if(params.isUseAliyunApi()){
|
||||
if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
|
||||
aliyunBo = aliyunService.addressStand(params.getAddress(), params.getCity());
|
||||
if(aliyunBo != null){
|
||||
String poi = ((aliyunBo.getPoi() == null ? "":aliyunBo.getPoi()) + (aliyunBo.getSubpoi() == null ? "":aliyunBo.getSubpoi())).replaceAll("\\s*", "");
|
||||
if(StringUtils.isNotBlank(poi)){
|
||||
addr.setKeywords(poi);
|
||||
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
|
||||
addr.setKeywords(poi);
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(aliyunBo.getCity())){
|
||||
addr.setSuggestCity(aliyunBo.getCity());
|
||||
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){
|
||||
String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length()));
|
||||
if(StringUtils.isNotBlank(str)){
|
||||
String newKeywords = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?"), str);
|
||||
if(StringUtils.isNotBlank(newKeywords)){
|
||||
addr.setKeywords(newKeywords);
|
||||
addr.setParsedPropertyName(newKeywords);
|
||||
aliyunBo.setRoomno(str.replace(newKeywords,""));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
addr.addKeyword(addr.getOrgAddress());
|
||||
if(params.isUseMap()){
|
||||
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP);
|
||||
}else{
|
||||
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB);
|
||||
}
|
||||
|
||||
/*if(jsonObj != null){
|
||||
//乡镇优化
|
||||
String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : "";
|
||||
if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){
|
||||
String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township);
|
||||
if(StringUtils.isNotBlank(town)){
|
||||
addr.setTownship(town);
|
||||
}
|
||||
}
|
||||
//道路优化(两边均去掉乡镇)
|
||||
String street = addr.getStreet();
|
||||
String streetNumber = addr.getStreetNumber();
|
||||
String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : "";
|
||||
String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : "";
|
||||
//去掉路名中含有乡镇名的
|
||||
if(StringUtils.isNotBlank(addr.getTownship())){
|
||||
street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street;
|
||||
streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf;
|
||||
}
|
||||
|
||||
//crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理
|
||||
if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){
|
||||
//Java解析出路名为空
|
||||
if(StringUtils.isBlank(street)){
|
||||
street = streetCrf;
|
||||
if(StringUtils.isNotBlank(streetNumberCrf)){
|
||||
streetNumber = streetNumberCrf;
|
||||
}
|
||||
}else{
|
||||
//CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则
|
||||
if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){
|
||||
street = streetCrf;
|
||||
}
|
||||
}
|
||||
addr.setStreet(street);
|
||||
addr.setStreetNumber(streetNumber);
|
||||
}
|
||||
//poi处理
|
||||
//没有匹配上库中小区时
|
||||
if(StringUtils.isBlank(matchedBuilding.getId())){
|
||||
String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") +
|
||||
(jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") +
|
||||
(jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : "");
|
||||
//去掉路名中含有乡镇名和路名 getParsedPropertyName
|
||||
if(StringUtils.isNotBlank(addr.getTownship())){
|
||||
poi = poi.replace(addr.getTownship(),"");
|
||||
}
|
||||
if(StringUtils.isNotBlank(addr.getStreet())){
|
||||
poi = poi.replace(addr.getStreet(),"");
|
||||
}
|
||||
addr.setParsedPropertyName(poi);
|
||||
}
|
||||
//楼栋、单元、室号
|
||||
String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", "");
|
||||
addr.setOrgBuildingNumber(houseNo);
|
||||
addr.setBuildingNumber(houseNo);
|
||||
addr.setOrgUnitNumber(cellNo);
|
||||
addr.setUnitNumber(cellNo);
|
||||
addr.setFloor(floorNo);
|
||||
addr.setOrgRoomNumber(roomNo);
|
||||
addr.setRoomNumber(roomNo);
|
||||
}*/
|
||||
matchedBuilding.setProvince(addr.getProvince());
|
||||
matchedBuilding.setSearchAddress(addr);
|
||||
matchedBuilding.setSuggestName(getSuggestName(matchedBuilding));
|
||||
matchedBuilding.setSuggestAddress(getSuggestAddress(matchedBuilding));
|
||||
if(params.isUseAliyunApi() && aliyunBo != null){
|
||||
if(params.isUseAliyunApi() && aliyunBo != null && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
|
||||
completBuildingField(matchedBuilding, aliyunBo, params);
|
||||
}
|
||||
Address tempAddr = matchedBuilding.getSearchAddress();
|
||||
if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
|
||||
AddressMatchMetaData meta = getMapMeta(addr);
|
||||
if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){
|
||||
matchedBuilding.setAddress(meta.getAddress());
|
||||
}
|
||||
}
|
||||
matchedBuilding.setSearchAddress(tempAddr);
|
||||
return matchedBuilding;
|
||||
}
|
||||
|
||||
private AddressMatchMetaData getMapMeta(Address tempAddr){
|
||||
tempAddr.addKeyword(tempAddr.getOrgAddress());
|
||||
List<AddressMatchMetaData> metaDatas = findMatchMetaData(tempAddr, null, MetaDataType.MAP, false);
|
||||
Map<String,List<AddressMatchMetaData>> metaDataMap = new HashMap<String,List<AddressMatchMetaData>>();
|
||||
if(StringUtils.isBlank(tempAddr.getDistrict()) && !metaDatas.isEmpty()){
|
||||
for(AddressMatchMetaData metaData:metaDatas){
|
||||
if(StringUtils.isNotBlank(metaData.getMapCity())){
|
||||
List<AddressMatchMetaData> metaList = metaDataMap.containsKey(metaData.getMapCity()) ? metaDataMap.get(metaData.getMapCity()) : new ArrayList<AddressMatchMetaData>();
|
||||
metaList.add(metaData);
|
||||
metaDataMap.put(metaData.getMapCity(),metaList);
|
||||
}
|
||||
}
|
||||
String tempCity = "";
|
||||
if(StringUtils.isNotBlank(tempAddr.getCity()) && metaDataMap.containsKey(tempAddr.getCity())){
|
||||
tempCity = tempAddr.getCity();
|
||||
}else{
|
||||
int size = 0;
|
||||
for(String key:metaDataMap.keySet()){
|
||||
if(metaDataMap.get(key).size() > size){
|
||||
size = metaDataMap.get(key).size();
|
||||
tempCity = key;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(tempCity)){
|
||||
AddressMatchMetaData meta = metaDataMap.get(tempCity).get(0);
|
||||
return meta;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用
|
||||
* @param address
|
||||
* @return
|
||||
*/
|
||||
/*private JSONObject getDataFromCRF(String address){
|
||||
HttpRequest request = new HttpRequest();
|
||||
Map<String,String> params = new HashMap<>();
|
||||
params.put("address",address);
|
||||
try {
|
||||
JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8");
|
||||
System.out.println(resp);
|
||||
return resp;
|
||||
}catch(Exception e){
|
||||
|
||||
}
|
||||
return null;
|
||||
}*/
|
||||
/**
|
||||
* 信息不全时使用阿里接口补全数据
|
||||
* @param matchedBuilding
|
||||
@@ -208,28 +364,52 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
}
|
||||
}
|
||||
Address addr = matchedBuilding.getSearchAddress();
|
||||
if(StringUtils.isBlank(addr.getProvince())){
|
||||
if(StringUtils.isNotBlank(aliyunBo.getProv())){
|
||||
addr.setProvince(formatProvince(aliyunBo.getProv()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(aliyunBo.getCity())){
|
||||
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
|
||||
}
|
||||
if(StringUtils.isNotBlank(aliyunBo.getDistrict())){
|
||||
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
|
||||
}
|
||||
addr.setTownship(aliyunBo.getTown());
|
||||
|
||||
/*if(StringUtils.isBlank(addr.getProvince())){
|
||||
addr.setProvince(formatProvince(aliyunBo.getProv()));
|
||||
}
|
||||
if(StringUtils.isBlank(addr.getCity())){
|
||||
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
|
||||
}
|
||||
|
||||
if(StringUtils.isBlank(addr.getDevelopmentZone())){
|
||||
addr.setDevelopmentZone(aliyunBo.getDevzone());
|
||||
}
|
||||
if(StringUtils.isBlank(addr.getDistrict())){
|
||||
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
|
||||
}
|
||||
if(StringUtils.isBlank(addr.getTownship())){
|
||||
addr.setTownship(aliyunBo.getTown());
|
||||
}*/
|
||||
if(StringUtils.isBlank(addr.getDevelopmentZone())){
|
||||
addr.setDevelopmentZone(aliyunBo.getDevzone());
|
||||
}
|
||||
addr.setOrgBuildingNumber(aliyunBo.getHouseno());
|
||||
|
||||
/*addr.setOrgBuildingNumber(aliyunBo.getHouseno());
|
||||
addr.setBuildingNumber(aliyunBo.getHouseno());
|
||||
addr.setOrgUnitNumber(aliyunBo.getCellno());
|
||||
addr.setUnitNumber(aliyunBo.getCellno());
|
||||
addr.setOrgRoomNumber(aliyunBo.getRoomno());
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
addr.setUnitNumber(aliyunBo.getCellno());*/
|
||||
/*if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){
|
||||
addr.setOrgRoomNumber(aliyunBo.getRoomno());
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
}else{
|
||||
addr.setRoomNumber(null);
|
||||
addr.setRoomNumber(null);
|
||||
}*/
|
||||
|
||||
/*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", "");
|
||||
String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", "");
|
||||
if(orgRoom!=null && orgRoom.equals(aliRoom)){
|
||||
addr.setRoomNumber(aliyunBo.getRoomno());
|
||||
}else{
|
||||
addr.setRoomNumber(null);
|
||||
}*/
|
||||
|
||||
if(StringUtils.isBlank(addr.getStreet())){
|
||||
addr.setStreet(aliyunBo.getRoad());
|
||||
@@ -318,6 +498,7 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
addr.setSuggestDistrict(params.getSuggestDistrict());
|
||||
addr.setSuggestName(params.getSuggestName());
|
||||
addr.setPreciseLevel(params.getPreciseLevel());
|
||||
addr.setAddressParserGoal(params.getAddressParserGoal());
|
||||
addr.setProvince(params.getProvince());
|
||||
addr.setDebugName(params.getDebugName());
|
||||
return AddressProcessors.process(addr);
|
||||
@@ -362,7 +543,7 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
* @param metaDataType
|
||||
* @return
|
||||
*/
|
||||
private List<AddressMatchMetaData> findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType){
|
||||
private List<AddressMatchMetaData> findMatchMetaData(final Address address, final BuildingType buildingType, MetaDataType metaDataType, boolean validCity){
|
||||
if(MetaDataType.MAP == metaDataType){
|
||||
List<String> keywords = address.getKeywordList();
|
||||
if(keywords.isEmpty()){
|
||||
@@ -375,7 +556,7 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
}
|
||||
keywords.add(standardAddress);
|
||||
}
|
||||
return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100);
|
||||
return addressRepoService.findAllByKeyword(address.getCity(), buildingType, keywords, address.getLng(), address.getLat(), 100, validCity);
|
||||
}else{
|
||||
return SEAWEED_LOCAL_CACHE.get(new MetaDataCacheKey(address.getCity(), buildingType, MetaDataType.CPDB));
|
||||
}
|
||||
@@ -517,20 +698,20 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
scoreResult.setOrgAddress(addr.getOrgAddress());
|
||||
if(buildingType != null) {
|
||||
if(MetaDataType.MAP == metaDataType){
|
||||
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP);
|
||||
metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB));
|
||||
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.MAP, true);
|
||||
metaDatas.addAll(findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true));
|
||||
}else{
|
||||
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB);
|
||||
metaDatas = findMatchMetaData(addr, buildingType, MetaDataType.CPDB, true);
|
||||
}
|
||||
} else {
|
||||
if(MetaDataType.MAP == metaDataType){
|
||||
metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP);
|
||||
metaDatas = findMatchMetaData(addr, null, MetaDataType.MAP, true);
|
||||
}
|
||||
for (BuildingType bt : addr.guessBuildType()) {
|
||||
if(metaDatas == null){
|
||||
metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB);
|
||||
metaDatas = findMatchMetaData(addr, bt, MetaDataType.CPDB, true);
|
||||
}else{
|
||||
metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB));
|
||||
metaDatas.addAll(findMatchMetaData(addr, bt, MetaDataType.CPDB, true));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -960,15 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
if(StringUtils.isNotBlank(address.getTownship())){
|
||||
formattedAddressBuilder.append(address.getTownship());
|
||||
}
|
||||
if(StringUtils.isNotBlank(building.getAddress())){
|
||||
Address orgAddr = new Address();
|
||||
orgAddr.setOrgAddress(building.getAddress());
|
||||
Address addr = AddressProcessors.process(orgAddr);
|
||||
if(StringUtils.isBlank(address.getStreet())){
|
||||
address.setStreet(addr.getStreet());
|
||||
}
|
||||
if(StringUtils.isBlank(address.getStreetNumber())){
|
||||
address.setStreetNumber(addr.getStreetNumber());
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getStreet())){
|
||||
formattedAddressBuilder.append(address.getStreet());
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){
|
||||
formattedAddressBuilder.append(address.getStreetNumber());
|
||||
}
|
||||
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
|
||||
formattedAddressBuilder.append(building.getAddress());
|
||||
}
|
||||
|
||||
if(StringUtils.isNotBlank(building.getSuggestName())){
|
||||
formattedAddressBuilder.append(building.getSuggestName());
|
||||
}
|
||||
@@ -985,18 +1175,18 @@ public class AddressParserServiceImpl implements AddressParserService {
|
||||
&& address.getStreetNumber().endsWith(address.getOrgBuildingNumber())){
|
||||
|
||||
}else{
|
||||
formattedAddressBuilder.append(address.getBuildingNumber());
|
||||
// formattedAddressBuilder.append(address.getBuildingNumber() + "¶°");
|
||||
// formattedAddressBuilder.append(address.getBuildingNumber());
|
||||
formattedAddressBuilder.append(address.getBuildingNumber() + "栋");
|
||||
}
|
||||
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getOrgUnitNumber())){
|
||||
formattedAddressBuilder.append(address.getUnitNumber());
|
||||
// formattedAddressBuilder.append(address.getUnitNumber() + "µ¥Ôª");
|
||||
// formattedAddressBuilder.append(address.getUnitNumber());
|
||||
formattedAddressBuilder.append(address.getUnitNumber() + "单元");
|
||||
}
|
||||
if(StringUtils.isNotBlank(address.getOrgRoomNumber())){
|
||||
formattedAddressBuilder.append(address.getRoomNumber());
|
||||
// formattedAddressBuilder.append(address.getRoomNumber() + "ÊÒ");
|
||||
if(StringUtils.isNotBlank(address.getRoomNumber())){
|
||||
// formattedAddressBuilder.append(address.getRoomNumber());
|
||||
formattedAddressBuilder.append(address.getRoomNumber() + "室");
|
||||
}
|
||||
address.setFormattedAddress(formattedAddressBuilder.toString());
|
||||
|
||||
|
||||
+40
-5
@@ -63,14 +63,18 @@ public class AddressFormatterFactory {
|
||||
});
|
||||
|
||||
|
||||
/*人民中路123-3501 -->人民中路123号3501室*/
|
||||
formatters.add(new AbstractAddressFormatter(Pattern.compile(".*[路|弄]\\d+-\\d{3,4}$")) {
|
||||
/*人民中路123-128 -->人民中路123号-3501号*/
|
||||
formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?\\d+-\\d{3,4}(号)?$")) {
|
||||
|
||||
final Pattern TARGET_PATTERN = Pattern.compile("(?<=(路|号))\\d+-\\d{3,4}$");
|
||||
final Pattern TARGET_PATTERN = Pattern.compile("(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?\\d+-\\d{3,4}(号)?$");
|
||||
@Override
|
||||
protected String doFormat(String address) {
|
||||
String targetStr = DataUtil.match(address, TARGET_PATTERN);
|
||||
return address.replace(targetStr, targetStr.replace("-", "号") + "室");
|
||||
if(targetStr.endsWith("号")){
|
||||
return address.replace(targetStr, targetStr.replace("-", "号-"));
|
||||
}else{
|
||||
return address.replace(targetStr, targetStr.replace("-", "号-") + "号");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
@@ -259,7 +263,38 @@ public class AddressFormatterFactory {
|
||||
@Override
|
||||
protected String doFormat(String address) {
|
||||
String matchStr = DataUtil.match(address, TARGET);
|
||||
String formatStr = matchStr + "号";
|
||||
String formatStr = matchStr.replaceAll("段-","段");
|
||||
if(!matchStr.endsWith("号")){
|
||||
formatStr += "号";
|
||||
}
|
||||
return address.replace(matchStr,formatStr);
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* 剑南路东段-12号-18号
|
||||
*/
|
||||
formatters.add(new AbstractAddressFormatter(Pattern.compile(".*(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))")) {
|
||||
final Pattern TARGET = Pattern.compile("(路|街|大道|巷|道|((零|一|二|三|四|五|六|七|八|九|十|东|南|西|北|复)[线环条甲])|(胡同))(交汇|.{0,3}段|.*弄)?((([零一二三四五六七八九十百千\\d])+弄)|([甲乙-]?\\d+[号](附\\d+(号)?(-\\d+(号)?)?)?([-][附付、]?\\d+(号(附近)?)?)?))");
|
||||
@Override
|
||||
protected String doFormat(String address) {
|
||||
String matchStr = DataUtil.match(address, TARGET);
|
||||
String formatStr = matchStr;
|
||||
if(!matchStr.endsWith("号")){
|
||||
formatStr += "号";
|
||||
}
|
||||
return address.replace(matchStr,formatStr);
|
||||
}
|
||||
});
|
||||
/**
|
||||
* 剑南路东段-12号-18号
|
||||
*/
|
||||
formatters.add(new AbstractAddressFormatter(Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))")) {
|
||||
final Pattern TARGET = Pattern.compile("(((路|街|大道|巷|道|段|附|付)-)|(-(附|付)))");
|
||||
@Override
|
||||
protected String doFormat(String address) {
|
||||
String matchStr = DataUtil.match(address, TARGET);
|
||||
String formatStr = matchStr.replace("-","");
|
||||
return address.replace(matchStr,formatStr);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -77,7 +77,27 @@ public class Address {
|
||||
private String debugName;
|
||||
|
||||
private AliyunBo aliyunBo;
|
||||
//使用CRF切分的地址
|
||||
private String crfAddress;
|
||||
|
||||
public String getCrfAddress() {
|
||||
return crfAddress;
|
||||
}
|
||||
|
||||
public void setCrfAddress(String crfAddress) {
|
||||
this.crfAddress = crfAddress;
|
||||
}
|
||||
|
||||
public AddressParserGoal getAddressParserGoal() {
|
||||
return addressParserGoal;
|
||||
}
|
||||
|
||||
public void setAddressParserGoal(AddressParserGoal addressParserGoal) {
|
||||
this.addressParserGoal = addressParserGoal;
|
||||
}
|
||||
|
||||
/**地址解析目的*/
|
||||
private AddressParserGoal addressParserGoal;
|
||||
public AliyunBo getAliyunBo() {
|
||||
return aliyunBo;
|
||||
}
|
||||
|
||||
+18
@@ -44,6 +44,24 @@ public class AddressMatchMetaData {
|
||||
private Double lat;
|
||||
private String map;
|
||||
|
||||
public String getMapProvince() {
|
||||
return mapProvince;
|
||||
}
|
||||
|
||||
public void setMapProvince(String mapProvince) {
|
||||
this.mapProvince = mapProvince;
|
||||
}
|
||||
|
||||
public String getMapCity() {
|
||||
return mapCity;
|
||||
}
|
||||
|
||||
public void setMapCity(String mapCity) {
|
||||
this.mapCity = mapCity;
|
||||
}
|
||||
|
||||
private String mapProvince;
|
||||
private String mapCity;
|
||||
public void addName(String name, float priority){
|
||||
if(StringUtils.isNotBlank(name)){
|
||||
String formatName = AddressParserUtil.standardAddressAndName(name);
|
||||
|
||||
+2
-6
@@ -64,9 +64,7 @@ public class Building {
|
||||
this.alias = d.getAlias();
|
||||
this.addrAlias = d.getAddrAlias();
|
||||
this.propertyType = d.getEstateType2();
|
||||
if(StringUtils.isNotBlank(d.getStreetCommunity())){
|
||||
this.township = d.getStreetCommunity();
|
||||
}
|
||||
this.township = d.getTownship();
|
||||
if(d.getLat() != null && d.getLng() != null){
|
||||
this.lat = d.getLat()/10000000000d;
|
||||
this.lng = d.getLng()/10000000000d;
|
||||
@@ -90,9 +88,7 @@ public class Building {
|
||||
this.lat = d.getLat2()/10000000000d;
|
||||
this.lng = d.getLng2()/10000000000d;
|
||||
}
|
||||
if(StringUtils.isNotBlank(d.getStreetCommunity())){
|
||||
this.township = d.getStreetCommunity();
|
||||
}
|
||||
this.township = d.getTownship();
|
||||
}
|
||||
|
||||
public Building() {
|
||||
|
||||
+8
@@ -25,7 +25,15 @@ public class MatchParams {
|
||||
private boolean useMap;
|
||||
/**是否使用阿里云接口*/
|
||||
private boolean useAliyunApi;
|
||||
//是否使用CRF算法
|
||||
private boolean useCrf;
|
||||
public boolean isUseCrf() {
|
||||
return useCrf;
|
||||
}
|
||||
|
||||
public void setUseCrf(boolean useCrf) {
|
||||
this.useCrf = useCrf;
|
||||
}
|
||||
public boolean isContainsLocation() {
|
||||
return containsLocation;
|
||||
}
|
||||
|
||||
+15
-6
@@ -50,18 +50,23 @@ public class CityProcessor extends AbstractAddressProcessor {
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(city)){
|
||||
if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){
|
||||
String formatCity = RegionData.formatCity(city);
|
||||
String possibleCity = null;
|
||||
String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? parseCityFromAddress(orgAddress, address) : null;
|
||||
String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) ? null : formatCity;
|
||||
/**如果省份和城市信息相互矛盾 则用区域推断城市*/
|
||||
if(StringUtils.isNotBlank(address.getProvince()) && !ProvinceData.belongTo(formatCity, address.getProvince())){
|
||||
List<String> possibleCities = speculateCityNameByRegion(address, city);
|
||||
if(isUnique(possibleCities)){
|
||||
formatCity = getFirst(possibleCities);
|
||||
confirmCity = formatCity;
|
||||
}
|
||||
}
|
||||
if(null != formatCity){
|
||||
city = formatCity;
|
||||
if(AddressParserGoal.Match.equals(address.getAddressParserGoal()) && (formatCity == null || (possibleCity!=null && !formatCity.equals(possibleCity)))){
|
||||
confirmCity = possibleCity;
|
||||
}
|
||||
if(null != confirmCity){
|
||||
city = confirmCity;
|
||||
address.setCity(city);
|
||||
address.setCityStatus(CityStatus.SURE);
|
||||
}
|
||||
@@ -98,7 +103,7 @@ public class CityProcessor extends AbstractAddressProcessor {
|
||||
address.setCity(city);
|
||||
}
|
||||
if(StringUtils.isNotBlank(city) && scoreRegion(city, orgAddress) > 0.4){
|
||||
orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?" , "");
|
||||
orgAddress = orgAddress.replaceFirst(".*" + address.getCity() + "[市县区州巿]?(?!(路|街|大道|巷|道|段))" , "");
|
||||
address.setAddress(orgAddress);
|
||||
address.setCityStatus(CityStatus.SURE);
|
||||
return RepeatStatus.NEXT;
|
||||
@@ -166,7 +171,7 @@ public class CityProcessor extends AbstractAddressProcessor {
|
||||
String street = getStreet(address);
|
||||
for(String city : allCity){
|
||||
if(street != null){
|
||||
MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)"), street);
|
||||
MatchResult matchResult = matchFirst(Pattern.compile(city + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street);
|
||||
if(matchResult !=null){
|
||||
address = address.replace(matchResult.matchContent, "");
|
||||
}
|
||||
@@ -199,6 +204,10 @@ public class CityProcessor extends AbstractAddressProcessor {
|
||||
}
|
||||
Map<String, List<String>> regionMap = RegionData.getAllRegions(city);
|
||||
for(String region : regionMap.keySet()){
|
||||
MatchResult matchResult = matchFirst(Pattern.compile(region + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|大街|支路|村)"), street);
|
||||
if(matchResult !=null){
|
||||
address = address.replace(matchResult.matchContent, "");
|
||||
}
|
||||
if(address.indexOf(region, index) > -1){
|
||||
if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){
|
||||
weight.score += 2;
|
||||
|
||||
+2
-1
@@ -2,6 +2,7 @@ package com.fangjia.service.address.processor;
|
||||
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.service.address.mode.Address;
|
||||
import com.fangjia.service.address.mode.AddressParserGoal;
|
||||
import com.fangjia.service.address.mode.CityStatus;
|
||||
import com.fangjia.service.address.mode.ProvinceData;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
@@ -24,7 +25,7 @@ public class ProvinceProcessor extends AbstractAddressProcessor {
|
||||
|
||||
String orgAddress = address.getAddress();
|
||||
String[] provinceInfo = getProvince(orgAddress);
|
||||
if(provinceInfo == null && address.getCity() != null){
|
||||
if(provinceInfo == null && address.getCity() != null && !AddressParserGoal.Match.equals(address.getAddressParserGoal())){
|
||||
provinceInfo = getProvince(address.getCity());
|
||||
}
|
||||
if (null != provinceInfo) {
|
||||
|
||||
+12
-4
@@ -18,6 +18,7 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
|
||||
static final Pattern NUMBER_END = Pattern.compile("(?<=\\D)\\d{3,4}$");
|
||||
static final Pattern HOUSE_NUMBER = Pattern.compile("\\d{3,4}");
|
||||
static final String HOUSE_UNITS_PATTERN = "(ÊÒ|·¿|ºÅ·¿|ºÅ(?!Ôº)|»§|ÆÌ)+";
|
||||
static final Pattern HOUSE_PATTERN_CASE02 = Pattern.compile("(\\d+(ÊÒ|·¿|ºÅ)?[¡¢]?)+$");
|
||||
|
||||
@Override
|
||||
public RepeatStatus process(Address address) {
|
||||
@@ -27,10 +28,17 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
|
||||
orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end);
|
||||
address.setOrgRoomNumber(matchResult.matchContent);
|
||||
address.setAddress(orgAddress);
|
||||
analyFloorUseRoomNumber(address);
|
||||
// analyFloorUseRoomNumber(address);
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
MatchResult matchResult2 = matchFirst(HOUSE_PATTERN_CASE02, orgAddress);
|
||||
if (matchResult2 != null) {
|
||||
orgAddress = removeSubstring(orgAddress, matchResult2.start, matchResult2.end);
|
||||
address.setOrgRoomNumber(matchResult2.matchContent);
|
||||
address.setAddress(orgAddress);
|
||||
// analyFloorUseRoomNumber(address);
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
|
||||
matchResult = matchFirst(HOUSE_PATTERN, orgAddress);
|
||||
if (matchResult != null) {
|
||||
orgAddress = removeSubstring(orgAddress, matchResult.start, matchResult.end);
|
||||
@@ -39,10 +47,10 @@ public class RoomNumberProcessor extends AbstractAddressProcessor {
|
||||
} else {
|
||||
address.setOrgRoomNumber(matchResult.matchContent + "ÊÒ");
|
||||
}
|
||||
analyFloorUseRoomNumber(address);
|
||||
// analyFloorUseRoomNumber(address);
|
||||
address.setAddress(orgAddress);
|
||||
}
|
||||
analyFloorUseRoomNumber(address);
|
||||
// analyFloorUseRoomNumber(address);
|
||||
removeHouseUnit(address);
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
|
||||
+1
@@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor {
|
||||
@Override
|
||||
public RepeatStatus process(Address address) {
|
||||
String orgAddr = address.getEffectiveAddress();
|
||||
address.setCrfAddress(address.getAddress());
|
||||
MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress());
|
||||
if (matchResult != null) {
|
||||
String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent);
|
||||
|
||||
+1
@@ -41,6 +41,7 @@ public class TooltipProcessor extends AbstractAddressProcessor {
|
||||
offset += matchResult.matchContent.length();
|
||||
if(matchFirst(AddressParserPatterns.STREET, tooltip) != null){
|
||||
address.setTooltip(tooltip);
|
||||
address.setStreet(getStreet(tooltip));
|
||||
} else if(containsDistrictName(tooltip)){
|
||||
address.setTooltip(tooltip);
|
||||
} else if(RegionData.isRegion(address.getCity(), tooltip)){
|
||||
|
||||
+10
@@ -1,6 +1,7 @@
|
||||
package com.fangjia.service.address.processor;
|
||||
|
||||
import com.fangjia.service.address.mode.Address;
|
||||
import com.fangjia.service.cnAnalyzer.Token;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
|
||||
import java.util.ArrayList;
|
||||
@@ -14,6 +15,15 @@ public class VillageNameProcessor extends AbstractAddressProcessor {
|
||||
@Override
|
||||
public AbstractAddressProcessor.RepeatStatus process(Address address) {
|
||||
String orgAddress = address.getAddress();
|
||||
if(address.getTokens()!=null){
|
||||
for(Token token:address.getTokens()){
|
||||
if("district".equals(token.getKeywordTypes())){
|
||||
address.setKeywords(token.getName());
|
||||
address.setAddress(orgAddress.replace(token.getName(),""));
|
||||
return RepeatStatus.NEXT;
|
||||
}
|
||||
}
|
||||
}
|
||||
List<String> names = tokenDistrictNames(orgAddress);
|
||||
List<String> kws = new ArrayList<String>();
|
||||
for(String name : names){
|
||||
|
||||
+76
-14
@@ -1,5 +1,7 @@
|
||||
package com.fangjia.service.api;
|
||||
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest;
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse;
|
||||
import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest;
|
||||
import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse;
|
||||
import com.fangjia.core.util.JsonUtils;
|
||||
@@ -13,6 +15,9 @@ import net.sf.json.JSONObject;
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.slf4j.Logger;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class AliyunServiceImpl implements AliyunService {
|
||||
|
||||
protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass());
|
||||
@@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService {
|
||||
if(StringUtils.isNotBlank(city)){
|
||||
request.setDefaultCity(city);
|
||||
}
|
||||
AliyunBo aliyunBo = new AliyunBo();
|
||||
try{
|
||||
StructureAddressResponse response = client.getAcsResponse(request);
|
||||
JSONObject responseJson = JSONObject.fromObject(response.getData());
|
||||
String value = (String) responseJson.get("structure");
|
||||
AliyunBo aliyunBo = new AliyunBo();
|
||||
for (String key:value.split("\t")){
|
||||
String[] arry = key.split("=");
|
||||
wrapField(arry[0],arry[1], aliyunBo);
|
||||
}
|
||||
return aliyunBo;
|
||||
}catch(Exception e){
|
||||
LOGGER.error("", e);
|
||||
return null;
|
||||
LOGGER.error("调用阿里标准化接口失败:", e);
|
||||
}
|
||||
try{
|
||||
GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest();
|
||||
divisionRequest.setServiceCode("addrp");
|
||||
divisionRequest.setAppKey(APPKEY);
|
||||
divisionRequest.setText(address);
|
||||
GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest);
|
||||
JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData());
|
||||
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
|
||||
String[] array = value.replace("\t", ";").split(";");
|
||||
if(array.length > 4){
|
||||
List<String> tmp = new ArrayList<String>();
|
||||
for(String str:array){
|
||||
if(str!=null && str.length()!=0){
|
||||
tmp.add(str);
|
||||
}
|
||||
}
|
||||
array= tmp.toArray(new String[4]);
|
||||
}
|
||||
aliyunBo.setProv(array[0]);
|
||||
aliyunBo.setCity(array[1]);
|
||||
aliyunBo.setDistrict(array[2]);
|
||||
aliyunBo.setTown(array[3]);
|
||||
}catch(Exception e){
|
||||
LOGGER.error("调用阿里行政区划识别接口失败", e);
|
||||
}
|
||||
return aliyunBo;
|
||||
}
|
||||
|
||||
private void wrapField(String field,String value, AliyunBo aliyunBo){
|
||||
@@ -81,26 +110,59 @@ public class AliyunServiceImpl implements AliyunService {
|
||||
}
|
||||
}
|
||||
public static void main(String[] args) throws ClientException {
|
||||
DefaultProfile profile = DefaultProfile.getProfile(
|
||||
String value = "";
|
||||
String[] array = value.replace("\t", ";").split(";");
|
||||
if(array.length > 4){
|
||||
List<String> tmp = new ArrayList<String>();
|
||||
for(String str:array){
|
||||
if(str!=null && str.length()!=0){
|
||||
tmp.add(str);
|
||||
}
|
||||
}
|
||||
array= tmp.toArray(new String[4]);
|
||||
}
|
||||
System.out.println(array[0]);
|
||||
System.out.println(array[1]);
|
||||
System.out.println(array[2]);
|
||||
System.out.println(array[3]);
|
||||
|
||||
/*DefaultProfile profile = DefaultProfile.getProfile(
|
||||
REGIONID,
|
||||
ACCESSKEYID,
|
||||
SECRET);
|
||||
IAcsClient client = new DefaultAcsClient(profile);
|
||||
StructureAddressRequest request = new StructureAddressRequest();
|
||||
// StructureAddressRequest request = new StructureAddressRequest();
|
||||
GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest();
|
||||
request.setServiceCode("addrp");
|
||||
request.setAppKey(APPKEY);
|
||||
request.setText("杭州市文一西路969号");
|
||||
request.setDefaultProvince("浙江省");
|
||||
request.setDefaultCity("杭州市");
|
||||
request.setDefaultDistrict("余杭区");
|
||||
StructureAddressResponse response = client.getAcsResponse(request);
|
||||
request.setText("天青路260号");
|
||||
// request.setDefaultProvince("浙江省");
|
||||
// request.setDefaultCity("杭州市");
|
||||
// request.setDefaultDistrict("余杭区");
|
||||
// StructureAddressResponse response = client.getAcsResponse(request);
|
||||
GetAddressDivisionCodeResponse response = client.getAcsResponse(request);
|
||||
JSONObject responseJson = JSONObject.fromObject(response.getData());
|
||||
String value = (String) responseJson.get("structure");
|
||||
JSONObject json = new JSONObject();
|
||||
System.out.println(responseJson);
|
||||
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
|
||||
String[] array = value.replace("\t", ";").split(";");
|
||||
if(array.length > 4){
|
||||
List<String> tmp = new ArrayList<String>();
|
||||
for(String str:array){
|
||||
if(str!=null && str.length()!=0){
|
||||
tmp.add(str);
|
||||
}
|
||||
}
|
||||
array= tmp.toArray(new String[4]);
|
||||
}
|
||||
System.out.println(array[0]);
|
||||
System.out.println(array[1]);
|
||||
System.out.println(array[2]);
|
||||
System.out.println(array[3]);*/
|
||||
/*JSONObject json = new JSONObject();
|
||||
for (String key:value.split("\t")){
|
||||
String[] arry = key.split("=");
|
||||
json.put(arry[0],arry[1]);
|
||||
}
|
||||
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));
|
||||
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,7 +68,7 @@ public interface AmapPoiService {
|
||||
* @param types
|
||||
* @return
|
||||
*/
|
||||
List<AmapPoi> search(String city, String keywords, Integer limit, String types);
|
||||
List<AmapPoi> search(String city, String keywords, Integer limit, String types, boolean validCity);
|
||||
|
||||
|
||||
/**
|
||||
|
||||
+12
-9
@@ -1,5 +1,6 @@
|
||||
package com.fangjia.service.api;
|
||||
|
||||
import com.fangjia.core.ProvinceData;
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.GeoUtils;
|
||||
@@ -258,14 +259,14 @@ public class AmapPoiServiceImpl implements AmapPoiService {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<AmapPoi> search(String city, String keywords, Integer limit, String types) {
|
||||
public List<AmapPoi> search(String city, String keywords, Integer limit, String types, boolean validCity) {
|
||||
Map<String, String> params = new HashMap<String, String>();
|
||||
String area = city;
|
||||
params.put("city", area);
|
||||
params.put("output", "json");
|
||||
params.put("keywords", keywords);
|
||||
params.put("key", "4ccfe3c898a9489884bd2bcb1ef6dcde");
|
||||
params.put("citylimit", "true");
|
||||
// params.put("citylimit", "true");
|
||||
params.put("offset", limit == null?"25":String.valueOf(limit));
|
||||
if(types != null){
|
||||
params.put("types", "120301&120302&120201&060101");
|
||||
@@ -351,15 +352,17 @@ public class AmapPoiServiceImpl implements AmapPoiService {
|
||||
cityname = poi.getString("cityname");
|
||||
}
|
||||
cityname = RegionData.formatCity(cityname);
|
||||
if(!city.equals(cityname)){
|
||||
if((city!=null && !city.equals(cityname) && validCity) || cityname == null){
|
||||
continue;
|
||||
}
|
||||
pname = ProvinceData.formatProvince(pname);
|
||||
AmapPoi amapPoi = new AmapPoi();
|
||||
amapPoi.setCity(city);
|
||||
amapPoi.setCity(cityname);
|
||||
amapPoi.setPname(pname);
|
||||
if(poi.containsKey("adname") && poi.get("adname") instanceof String){
|
||||
adname = poi.getString("adname");
|
||||
String region = RegionData.formatRegion(city, adname);
|
||||
if(RegionData.isRegion(city, region)){
|
||||
String region = RegionData.formatRegion(cityname, adname);
|
||||
if(RegionData.isRegion(cityname, region)){
|
||||
amapPoi.setRegion(region);
|
||||
}
|
||||
}
|
||||
@@ -380,8 +383,8 @@ public class AmapPoiServiceImpl implements AmapPoiService {
|
||||
amapPoi.setAlias(alias);
|
||||
}
|
||||
if(poi.containsKey("adname") && poi.get("adname") instanceof String){
|
||||
String region = RegionData.formatRegion(city, poi.getString("adname"));
|
||||
if(RegionData.isRegion(city, region)){
|
||||
String region = RegionData.formatRegion(cityname, poi.getString("adname"));
|
||||
if(RegionData.isRegion(cityname, region)){
|
||||
amapPoi.setRegion(region);
|
||||
}
|
||||
}
|
||||
@@ -391,7 +394,7 @@ public class AmapPoiServiceImpl implements AmapPoiService {
|
||||
.replace(cityname, "").replace(adname, "");
|
||||
amapPoi.setAddress(AddressParserUtil.optimizeAddress(address));
|
||||
}
|
||||
amapPoi.setCity(city);
|
||||
amapPoi.setCity(cityname);
|
||||
if(poi.containsKey("location") && poi.get("location") instanceof String){
|
||||
String[] location = poi.getString("location").split(",");
|
||||
amapPoi.setLng(Double.valueOf(location[0]));
|
||||
|
||||
+6
-4
@@ -2495,10 +2495,12 @@ public class DistrictServiceImpl implements DistrictService {
|
||||
params.add(new QueryParam("visible", 0));
|
||||
params.add(new QueryParam("city", city));
|
||||
params.add(new QueryParam("category", "district"));
|
||||
params.add(new QueryParam("keyword", keyword));
|
||||
Query query = boostQueryDistrictPolicy.create(ServiceUtil.removeAndGetKeyword(params));
|
||||
if (query != null) {
|
||||
params.add(new QueryParam(QueryParam.SR_PARAM_NAME, new SrQuery(query)));
|
||||
if (StringUtils.isNotBlank(keyword)) {
|
||||
BooleanQuery query = new BooleanQuery();
|
||||
Query q1 = new TermQuery(new Term("keyword", keyword));
|
||||
q1.setBoost(0.99f);
|
||||
query.add(q1, Occur.SHOULD);
|
||||
params.add(new QueryParam("$sr", new SrQuery(query)));
|
||||
}
|
||||
return districtDao.findAllByParams(params, limit);
|
||||
}
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ public interface AddressRepoService {
|
||||
* @param max
|
||||
* @return
|
||||
*/
|
||||
List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max);
|
||||
List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max, boolean validCity);
|
||||
|
||||
/**
|
||||
*
|
||||
|
||||
+8
-2
@@ -87,7 +87,7 @@ public class AddressRepoServiceImpl implements AddressRepoService {
|
||||
return addrs;
|
||||
}
|
||||
|
||||
public List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max){
|
||||
public List<AddressMatchMetaData> findAllByKeyword(String city,BuildingType buildingType , List<String> keywords, Double lng, Double lat, int max, boolean validCity){
|
||||
String types = null;
|
||||
String tag = null;
|
||||
String category = null;
|
||||
@@ -104,7 +104,11 @@ public class AddressRepoServiceImpl implements AddressRepoService {
|
||||
tag = "°Ù»õÉ̳¡";
|
||||
category = "×ÛºÏÉ̳¡";
|
||||
}
|
||||
List<AmapPoi> pois = amapPoiService.search(city, StringUtils.join(keywords, "|"), 20, types);
|
||||
List<AmapPoi> pois = new ArrayList<AmapPoi>();
|
||||
for(String keyword:keywords){
|
||||
List<AmapPoi> poiList = amapPoiService.search(city, keyword, 20, types, validCity);
|
||||
pois.addAll(poiList);
|
||||
}
|
||||
List<AddressMatchMetaData> matchMetaDatas = new ArrayList<AddressMatchMetaData>(pois.size());
|
||||
for(AmapPoi poi: pois){
|
||||
AddressMatchMetaData data = new AddressMatchMetaData();
|
||||
@@ -121,6 +125,8 @@ public class AddressRepoServiceImpl implements AddressRepoService {
|
||||
data.setType(poi.getType());
|
||||
data.setMap("¸ßµÂ");
|
||||
data.setCity(city);
|
||||
data.setMapCity(poi.getCity());
|
||||
data.setMapProvince(poi.getPname());
|
||||
if(poi.getAlias() != null){
|
||||
for(String alia : poi.getAlias()){
|
||||
data.addName(alia, MatchText.ALIAS_PRIORITY);
|
||||
|
||||
@@ -284,5 +284,12 @@ public interface AddressService {
|
||||
* @return
|
||||
*/
|
||||
AddressCheckTruth aroundConsumeGrade(String city, String address);
|
||||
|
||||
|
||||
/**
|
||||
* 量地址相似度评级
|
||||
* @param result1
|
||||
* @param result2
|
||||
* @return 返回是否同一地址、输出级别(哪一级),相似度得分
|
||||
*/
|
||||
Map<String,Object> addressSimilarity(String result1, String result2);
|
||||
}
|
||||
|
||||
+184
-23
@@ -5,18 +5,7 @@ import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.net.URLEncoder;
|
||||
import java.text.DecimalFormat;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.Comparator;
|
||||
import java.util.Date;
|
||||
import java.util.HashMap;
|
||||
import java.util.Iterator;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.LinkedHashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.Callable;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@@ -25,6 +14,9 @@ import java.util.concurrent.Future;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import com.fangjia.core.util.*;
|
||||
import com.fangjia.service.address.AddressParserUtil;
|
||||
import com.fangjia.service.address.mode.MatchResult;
|
||||
import net.sf.json.JSONArray;
|
||||
import net.sf.json.JSONObject;
|
||||
|
||||
@@ -49,18 +41,8 @@ import org.sothis.core.util.Closure;
|
||||
|
||||
import com.fangjia.core.BlockData;
|
||||
import com.fangjia.core.RegionData;
|
||||
import com.fangjia.core.util.AddressParser;
|
||||
import com.fangjia.core.util.AddressParserUtils;
|
||||
import com.fangjia.core.util.AddressResult;
|
||||
import com.fangjia.core.util.BeanUtils;
|
||||
import com.fangjia.core.util.CacheUtils;
|
||||
import com.fangjia.core.util.CacheUtils.CachePolicyEnum;
|
||||
import com.fangjia.core.util.DataUtil;
|
||||
import com.fangjia.core.util.DateUtil;
|
||||
import com.fangjia.core.util.GeoUtils;
|
||||
import com.fangjia.core.util.GeoUtils.GaussSphere;
|
||||
import com.fangjia.core.util.HttpRequest;
|
||||
import com.fangjia.core.util.LoggerFactory;
|
||||
import com.fangjia.dal.QueryParam;
|
||||
import com.fangjia.dal.SdQuery;
|
||||
import com.fangjia.dal.SrQuery;
|
||||
@@ -5074,7 +5056,186 @@ public class AddressServiceImpl implements AddressService {
|
||||
}
|
||||
return checkTr;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public Map<String, Object> addressSimilarity(String result1, String result2) {
|
||||
JSONObject json1 = (JSONObject) JSONObject.fromObject(result1).get("result");
|
||||
JSONObject json2 = (JSONObject) JSONObject.fromObject(result2).get("result");
|
||||
Map<String,Object> resultMap = new HashMap<String, Object>();
|
||||
//判断是否同一地址
|
||||
boolean sameAddress = validSameAddress(json1, json2);
|
||||
String formatAddress1_pinyin = AddressParserUtil.toHanYuPinYin(getFromJson(json1, "formattedAddress"));
|
||||
String formatAddress2_pinyin = AddressParserUtil.toHanYuPinYin(getFromJson(json2,"formattedAddress"));
|
||||
double similar = AddressParserUtil.getSimilarityRatio(formatAddress1_pinyin, formatAddress2_pinyin);
|
||||
String level = getAddressSimilarityLevel(json1, json2);
|
||||
resultMap.put("isSameAddress", sameAddress);
|
||||
// resultMap.put("score", similar);
|
||||
resultMap.put("level", level);
|
||||
resultMap.put("formattedAddress_1", getFromJson(json1,"formattedAddress"));
|
||||
resultMap.put("formattedAddress_2", getFromJson(json2,"formattedAddress"));
|
||||
return resultMap;
|
||||
}
|
||||
|
||||
private boolean validSameAddress(JSONObject json1, JSONObject json2){
|
||||
//标准化后地址完全一样
|
||||
if(getFromJson(json1, "formattedAddress").equals(getFromJson(json2, "formattedAddress"))){
|
||||
return true;
|
||||
}
|
||||
//判断判断省、市、区、乡镇、楼栋、单元、室号是否相同
|
||||
if(!verifySameField(getFromJson(json1,"province"), getFromJson(json2,"province"), false)
|
||||
|| !verifySameField(getFromJson(json1,"city"), getFromJson(json2,"city"), false)
|
||||
|| !verifySameField(getFromJson(json1,"district"), getFromJson(json2,"district"), false)
|
||||
|| !verifySameField(getFromJson(json1,"township"), getFromJson(json2,"township"), false)
|
||||
|| !verifySameField(getFromJson(json1,"buildingNumber"), getFromJson(json2,"buildingNumber"), true)
|
||||
|| !verifySameField(getFromJson(json1,"unitNumber"), getFromJson(json2,"unitNumber"), true)
|
||||
|| !verifySameField(getFromJson(json1,"roomNumber"), getFromJson(json2,"roomNumber"), true)){
|
||||
return false;
|
||||
}
|
||||
|
||||
String street1 = (StringUtils.isNotBlank(getFromJson(json1,"street")) ? getFromJson(json1,"street") : "") + (StringUtils.isNotBlank(getFromJson(json1,"streetNumber")) ? getFromJson(json1,"streetNumber") : "");
|
||||
String street2 = (StringUtils.isNotBlank(getFromJson(json2,"street")) ? getFromJson(json2,"street") : "") + (StringUtils.isNotBlank(getFromJson(json2,"streetNumber")) ? getFromJson(json2,"streetNumber") : "");
|
||||
String name1 = StringUtils.isNotBlank(getFromJson(json1,"name")) ? getFromJson(json1,"name") : "";
|
||||
String name2 = StringUtils.isNotBlank(getFromJson(json2,"name")) ? getFromJson(json2,"name") : "";
|
||||
//两个地址中路名+路号和poi全部为空时返回true
|
||||
if(StringUtils.isBlank(street1) && StringUtils.isBlank(street2) && StringUtils.isBlank(name1) && StringUtils.isBlank(name2)){
|
||||
return true;
|
||||
}
|
||||
//同一条地址里路名和poi不能全为空而另一条至少有一项有值
|
||||
if((StringUtils.isBlank(street1) && StringUtils.isBlank(name1)) || (StringUtils.isBlank(street2) && StringUtils.isBlank(name2))){
|
||||
return false;
|
||||
}
|
||||
//poi有任意一个为空则路名、路号必须完全一致
|
||||
if(StringUtils.isBlank(name1) || StringUtils.isBlank(name2)){
|
||||
if(StringUtils.isNotBlank(getFromJson(json1,"street")) && getFromJson(json1,"street").equals(getFromJson(json2,"street"))
|
||||
&& StringUtils.isNotBlank(getFromJson(json1,"streetNumber")) && getFromJson(json1,"streetNumber").equals(getFromJson(json2,"streetNumber"))){
|
||||
return true;
|
||||
}else{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
//poi全不为空时,路名+路号一致(可为空),poi处理后一致
|
||||
if(verifySameField(street1, street2, false) && verifySamePoi(name1, name2)){
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private String getFromJson(JSONObject json, String key){
|
||||
if(json.containsKey(key)){
|
||||
return json.getString(key);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
/**
|
||||
* 获取两地址的相似度评级(相同数据的最小颗粒度)
|
||||
* @return
|
||||
*/
|
||||
private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){
|
||||
String level = null;
|
||||
LinkedList<String> list = new LinkedList<String>(Arrays.asList("province", "city", "district", "township", "street", "streetNumber"
|
||||
, "name", "buildingNumber", "unitNumber", "roomNumber"));
|
||||
for(String key:list){
|
||||
if(continueJudgeValue(json1, json2, key) == 3){
|
||||
return level;
|
||||
}else{
|
||||
level = continueJudgeValue(json1, json2, key) == 1 ? key : (level == null ? null : level);
|
||||
}
|
||||
}
|
||||
return level;
|
||||
}
|
||||
|
||||
/**
|
||||
* 判断level时是否继续往下判断
|
||||
* @param json1
|
||||
* @param json2
|
||||
* @param key
|
||||
* @return 1 继续判断并且level有值 2继续判断level没值 3 停止
|
||||
*/
|
||||
private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){
|
||||
if("buildingNumber".equals(key) || "unitNumber".equals(key) || "roomNumber".equals(key)){
|
||||
if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))
|
||||
&& getFromJson(json1,key).equals(getFromJson(json2,key))){
|
||||
return 1;
|
||||
}else{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){
|
||||
if("name".equals(key)){
|
||||
return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3;
|
||||
}else{
|
||||
if(getFromJson(json1,key).equals(getFromJson(json2,key))){
|
||||
return 1;
|
||||
}else{
|
||||
return 3;
|
||||
}
|
||||
}
|
||||
}
|
||||
return 2;
|
||||
}
|
||||
/**
|
||||
* 判断省、市、区、乡镇、楼栋、单元、室号是否相同(两者全为空时返回true)
|
||||
* @param str1
|
||||
* @param str2
|
||||
* @param flag true时严格判断 两者完全一致返回true(全为空或者全部有值并且值一致)
|
||||
* false时 两者完全一致或一方为空返回true(全为空或者全部有值并且值一致或任一方值为空)
|
||||
* @return
|
||||
*/
|
||||
private boolean verifySameField(String str1, String str2,boolean flag){
|
||||
if(flag){
|
||||
if(StringUtils.isBlank(str1) && StringUtils.isBlank(str2)){
|
||||
return true;
|
||||
}
|
||||
return StringUtils.isNotBlank(str1) ? str1.equals(str2) : false;
|
||||
}else{
|
||||
if(StringUtils.isBlank(str1) || StringUtils.isBlank(str2)){
|
||||
return true;
|
||||
}
|
||||
return str1.equals(str2);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* //判断是否同一个poi(全部有数据时)
|
||||
* @param name1
|
||||
* @param name2
|
||||
* @return
|
||||
*/
|
||||
private boolean verifySamePoi(String name1, String name2){
|
||||
if(StringUtils.isBlank(name1) && StringUtils.isBlank(name2)){
|
||||
return false;
|
||||
}
|
||||
if(name1.equals(name2)){
|
||||
return true;
|
||||
}
|
||||
//分期
|
||||
String name1_stage = AddressParserUtil.match(AddressParserPatterns.STAGE_PATTERN,name1);
|
||||
String name2_stage = AddressParserUtil.match(AddressParserPatterns.STAGE_PATTERN,name2);
|
||||
//分区
|
||||
String name1_partition = AddressParserUtil.match(AddressParserPatterns.PARTITION_PATTERN,name1);
|
||||
String name2_partition = AddressParserUtil.match(AddressParserPatterns.PARTITION_PATTERN,name2);
|
||||
//数字
|
||||
String name1_number = AddressParserUtil.match(Pattern.compile("[\\d一二三四五六七八九十千百万]+"),name1);
|
||||
String name2_number = AddressParserUtil.match(Pattern.compile("[\\d一二三四五六七八九十千百万]+"),name2);
|
||||
if(!verifySameField(name1_stage, name2_stage, true) || !verifySameField(name1_partition, name2_partition, true)|| !verifySameField(name1_number, name2_number, true)){
|
||||
return false;
|
||||
}
|
||||
return isSameName(name1, name2);
|
||||
}
|
||||
|
||||
private boolean isSameName(String name1, String name2){
|
||||
String name_1_pinyin = AddressParserUtil.toHanYuPinYin(name1);
|
||||
String name_2_pinyin = AddressParserUtil.toHanYuPinYin(name2);
|
||||
//拼音或文本数据包含并且
|
||||
boolean pinYinContains = (name_1_pinyin.contains(name_2_pinyin) || name_2_pinyin.contains(name_1_pinyin));
|
||||
if(pinYinContains){
|
||||
double similar = AddressParserUtil.getSimilarityRatio(name_1_pinyin, name_2_pinyin);
|
||||
if(similar > 0.5){
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
/**
|
||||
* ÉèÖÃÇøÓòÏû·ÑÖ¸Êý
|
||||
* @param checkTr
|
||||
|
||||
@@ -29,7 +29,7 @@
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-core</artifactId>
|
||||
<version>0.2.0-SNAPSHOT</version>
|
||||
<version>0.2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>com.fangjia</groupId>
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
FROM 192.168.0.19:9012/tomcat6:latest
|
||||
COPY target/*.war /home/goojia/webapps/ROOT.war
|
||||
@@ -51,7 +51,7 @@
|
||||
<dependency>
|
||||
<groupId>org.sothis</groupId>
|
||||
<artifactId>sothis-web</artifactId>
|
||||
<version>0.2.0-SNAPSHOT</version>
|
||||
<version>0.2.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework</groupId>
|
||||
|
||||
+2
-8
@@ -22,14 +22,8 @@ public class DistrictController {
|
||||
private DistrictService districtService;
|
||||
|
||||
public Object loadDistrictAction(DistrictModel model, HttpServletRequest request){
|
||||
String url = openSearchService.createQueryUrl(model.getCityName(), model.getKeywords(), false);
|
||||
String content = openSearchService.findResultByUrl(url);
|
||||
List<String> regions = districtService.getRegionByCity(model.getCityName());
|
||||
Map<String, List<DistrictBo>> map = openSearchService.parseResult(content, regions, model.getKeywords());
|
||||
if(map.size() == 0){
|
||||
List<District> districts = districtService.findNewDistrict(model.getCityName(), model.getKeywords());
|
||||
map = openSearchService.createResult(districts);
|
||||
}
|
||||
List<District> districts = districtService.findDistrictByKeyword(model.getCityName(), model.getKeywords(), 10);
|
||||
Map<String, List<DistrictBo>> map = openSearchService.createResult(districts);
|
||||
JsonModel ret = new JsonModel();
|
||||
ret.setJsonModel(map);
|
||||
return ret;
|
||||
|
||||
@@ -155,11 +155,11 @@ public class BaseModel extends ModelAndViewSupport {
|
||||
}
|
||||
|
||||
public final String getRoot() {
|
||||
return (CONFIG.isDevelopmentEnvironment() ? "/geluke" : "");
|
||||
return "http://geluke.fangjia.com";
|
||||
}
|
||||
|
||||
public final String getWwwUrl() {
|
||||
return WWW_URL;
|
||||
return "http://geluke.fangjia.com";
|
||||
}
|
||||
|
||||
public final Date getNow() {
|
||||
@@ -270,7 +270,7 @@ public class BaseModel extends ModelAndViewSupport {
|
||||
}
|
||||
|
||||
public final String getCityUrl() {
|
||||
return CONFIG.getCityDomainName() + getRoot();
|
||||
return "http://geluke.fangjia.com";
|
||||
}
|
||||
|
||||
public final String getDomain() {
|
||||
|
||||
@@ -17,16 +17,32 @@ fangjia.auto.url=${fangjia.auto.host}:${fangjia.auto.port}/
|
||||
fangjia.file.mongo.host=fs.mongo.db.fangjia.com
|
||||
fangjia.file.mongo.port=27017
|
||||
|
||||
#fangjia.log.mongo.host=log.mongo.db.fangjia.com
|
||||
#fangjia.log.mongo.port=27017
|
||||
|
||||
fangjia.log.mongo.host=log.mongo.db.fangjia.com
|
||||
fangjia.log.mongo.port=27017
|
||||
fangjia.log.mongo.port=52002
|
||||
fangjia.log.mongodb.mongoUri=mongodb://${fangjia.log.mongo.host}:${fangjia.log.mongo.port}/
|
||||
|
||||
fangjia.mongo.host=mongo.db.fangjia.com
|
||||
fangjia.mongo.port=27017
|
||||
fangjia.mongo.port=28017
|
||||
fangjia.secondary.mongo.host=secondary.mongo.db.fangjia.com
|
||||
fangjia.secondary.mongo.port=28017
|
||||
fangjia.mongodb.mongoUri=mongodb://${fangjia.mongo.host}:${fangjia.mongo.port},${fangjia.secondary.mongo.host}:${fangjia.secondary.mongo.port}/
|
||||
|
||||
|
||||
fangjia.fs.mongo.host=fs.mongo.db.fangjia.com
|
||||
fangjia.fs.mongo.port=27017
|
||||
fangjia.fs.mongodb.mongoUri=mongodb://${fangjia.fs.mongo.host}:${fangjia.fs.mongo.port}/
|
||||
|
||||
fangjia.seabird.mongodb.mongoUri=mongodb://developer:G00IDCDEV12385@${fangjia.mongo.host}:${fangjia.mongo.port},${fangjia.secondary.mongo.host}:${fangjia.secondary.mongo.port}/
|
||||
|
||||
|
||||
fangjia.knowbase.host=knowbase.db.fangjia.com
|
||||
fangjia.knowbase.port=7180
|
||||
fangjia.knowbase.url=${fangjia.knowbase.host}:${fangjia.knowbase.port}/
|
||||
|
||||
|
||||
fangjia.environment=development
|
||||
#fangjia.environment=production
|
||||
|
||||
|
||||
@@ -18,8 +18,15 @@
|
||||
</bean>
|
||||
|
||||
<import resource="classpath:spring-service.xml" />
|
||||
|
||||
|
||||
<bean id="seabirdMongo" class="com.mongodb.MongoClient">
|
||||
<constructor-arg ref="seabirdMongoUri" />
|
||||
</bean>
|
||||
<bean id="seabirdMongoUri" class="com.mongodb.MongoClientURI">
|
||||
<constructor-arg type="java.lang.String" value="${fangjia.seabird.mongodb.mongoUri}"/>
|
||||
</bean>
|
||||
<bean id="seaweedDao" class="com.fangjia.dal.seaweed.dao.SeaweedDaoImpl">
|
||||
<property name="mongo" ref="seabirdMongo" />
|
||||
</bean>
|
||||
<bean id="dataStore" class="com.fangjia.datastore.driver.DataStore">
|
||||
<constructor-arg value="${fangjia.datastore.host}" />
|
||||
<constructor-arg value="${fangjia.datastore.port}" />
|
||||
@@ -29,20 +36,27 @@
|
||||
<constructor-arg value="${fangjia.seaweed.host}" />
|
||||
<constructor-arg value="${fangjia.seaweed.port}" />
|
||||
</bean>
|
||||
|
||||
<bean id="mongo" class="com.mongodb.Mongo">
|
||||
<constructor-arg index="0" value="${fangjia.mongo.host}" type="java.lang.String"/>
|
||||
<constructor-arg index="1" value="${fangjia.mongo.port}" type="int"/>
|
||||
|
||||
<bean id="mongoUri" class="com.mongodb.MongoClientURI">
|
||||
<constructor-arg type="java.lang.String" value="${fangjia.mongodb.mongoUri}"/>
|
||||
</bean>
|
||||
<bean id="mongo" class="com.mongodb.MongoClient">
|
||||
<constructor-arg ref="mongoUri" />
|
||||
</bean>
|
||||
|
||||
<bean id="logMongo" class="com.mongodb.Mongo">
|
||||
<constructor-arg index="0" value="${fangjia.log.mongo.host}" type="java.lang.String"/>
|
||||
<constructor-arg index="1" value="${fangjia.log.mongo.port}" type="int"/>
|
||||
|
||||
<bean id="logMongoUri" class="com.mongodb.MongoClientURI">
|
||||
<constructor-arg type="java.lang.String" value="${fangjia.log.mongodb.mongoUri}"/>
|
||||
</bean>
|
||||
|
||||
<bean id="fileMongo" class="com.mongodb.Mongo">
|
||||
<constructor-arg index="0" value="${fangjia.file.mongo.host}" type="java.lang.String"/>
|
||||
<constructor-arg index="1" value="${fangjia.file.mongo.port}" type="int"/>
|
||||
<bean id="logMongo" class="com.mongodb.MongoClient">
|
||||
<constructor-arg ref="logMongoUri" />
|
||||
</bean>
|
||||
|
||||
<bean id="fileMongoUri" class="com.mongodb.MongoClientURI">
|
||||
<constructor-arg type="java.lang.String" value="${fangjia.fs.mongodb.mongoUri}"/>
|
||||
</bean>
|
||||
<bean id="fileMongo" class="com.mongodb.MongoClient">
|
||||
<constructor-arg ref="fileMongoUri" />
|
||||
</bean>
|
||||
|
||||
<context:component-scan base-package="com.fangjia.dal.datastore.dao"
|
||||
|
||||
@@ -25,12 +25,12 @@
|
||||
</filter-class>
|
||||
</filter>
|
||||
|
||||
<filter>
|
||||
<filter-name>AccessLogFilter</filter-name>
|
||||
<filter-class>
|
||||
com.fangjia.geluke.filters.AccessLogFilter
|
||||
</filter-class>
|
||||
</filter>
|
||||
<!-- <filter>-->
|
||||
<!-- <filter-name>AccessLogFilter</filter-name>-->
|
||||
<!-- <filter-class>-->
|
||||
<!-- com.fangjia.geluke.filters.AccessLogFilter-->
|
||||
<!-- </filter-class>-->
|
||||
<!-- </filter>-->
|
||||
|
||||
<filter>
|
||||
<filter-name>SpiderValidateFilter</filter-name>
|
||||
@@ -47,20 +47,20 @@
|
||||
</init-param>
|
||||
</filter>
|
||||
|
||||
<filter>
|
||||
<filter-name>AnalyticsFilter</filter-name>
|
||||
<filter-class>
|
||||
org.springframework.web.filter.DelegatingFilterProxy
|
||||
</filter-class>
|
||||
<init-param>
|
||||
<param-name>targetBeanName</param-name>
|
||||
<param-value>analyticsFilter</param-value>
|
||||
</init-param>
|
||||
<init-param>
|
||||
<param-name>targetFilterLifecycle</param-name>
|
||||
<param-value>true</param-value>
|
||||
</init-param>
|
||||
</filter>
|
||||
<!-- <filter>-->
|
||||
<!-- <filter-name>AnalyticsFilter</filter-name>-->
|
||||
<!-- <filter-class>-->
|
||||
<!-- org.springframework.web.filter.DelegatingFilterProxy-->
|
||||
<!-- </filter-class>-->
|
||||
<!-- <init-param>-->
|
||||
<!-- <param-name>targetBeanName</param-name>-->
|
||||
<!-- <param-value>analyticsFilter</param-value>-->
|
||||
<!-- </init-param>-->
|
||||
<!-- <init-param>-->
|
||||
<!-- <param-name>targetFilterLifecycle</param-name>-->
|
||||
<!-- <param-value>true</param-value>-->
|
||||
<!-- </init-param>-->
|
||||
<!-- </filter>-->
|
||||
|
||||
<filter>
|
||||
<filter-name>sothis</filter-name>
|
||||
@@ -80,21 +80,21 @@
|
||||
<url-pattern>/*</url-pattern>
|
||||
<dispatcher>REQUEST</dispatcher>
|
||||
</filter-mapping>
|
||||
<filter-mapping>
|
||||
<filter-name>AccessLogFilter</filter-name>
|
||||
<url-pattern>/*</url-pattern>
|
||||
<dispatcher>REQUEST</dispatcher>
|
||||
</filter-mapping>
|
||||
<!-- <filter-mapping>-->
|
||||
<!-- <filter-name>AccessLogFilter</filter-name>-->
|
||||
<!-- <url-pattern>/*</url-pattern>-->
|
||||
<!-- <dispatcher>REQUEST</dispatcher>-->
|
||||
<!-- </filter-mapping>-->
|
||||
<filter-mapping>
|
||||
<filter-name>SpiderValidateFilter</filter-name>
|
||||
<url-pattern>/*</url-pattern>
|
||||
<dispatcher>REQUEST</dispatcher>
|
||||
</filter-mapping>
|
||||
<filter-mapping>
|
||||
<filter-name>AnalyticsFilter</filter-name>
|
||||
<url-pattern>/*</url-pattern>
|
||||
<dispatcher>REQUEST</dispatcher>
|
||||
</filter-mapping>
|
||||
<!-- <filter-mapping>-->
|
||||
<!-- <filter-name>AnalyticsFilter</filter-name>-->
|
||||
<!-- <url-pattern>/*</url-pattern>-->
|
||||
<!-- <dispatcher>REQUEST</dispatcher>-->
|
||||
<!-- </filter-mapping>-->
|
||||
<filter-mapping>
|
||||
<filter-name>sothis</filter-name>
|
||||
<url-pattern>/*</url-pattern>
|
||||
|
||||
@@ -23,58 +23,55 @@ id="changecity">
|
||||
<li>
|
||||
<div class="left">华东城市</div>
|
||||
<div class="right">
|
||||
<a name="shanghai" href="http://sh.${domain}${root}/">上海</a>
|
||||
<a name="hangzhou" href="http://hz.${domain}${root}/">杭州</a>
|
||||
<a name="nanjing" href="http://nj.${domain}${root}/">南京</a>
|
||||
<a name="ningbo" href="http://nb.${domain}${root}/">宁波</a>
|
||||
<a name="suzhou" href="http://suz.${domain}${root}/">苏州</a>
|
||||
<a name="wuxi" href="http://wx.${domain}${root}/">无锡</a>
|
||||
<a name="jinan" href="http://jn.${domain}${root}/">济南</a>
|
||||
<a name="qingdao" href="http://qd.${domain}${root}/">青岛</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E4%B8%8A%E6%B5%B7">上海</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%9D%AD%E5%B7%9E">杭州</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%8D%97%E4%BA%AC">南京</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%AE%81%E6%B3%A2">宁波</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%97%A0%E9%94%A1">无锡</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%B5%8E%E5%8D%97">济南</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%9D%92%E5%B2%9B">青岛</a>
|
||||
</div>
|
||||
</li>
|
||||
<li>
|
||||
<div class="left">华北东北</div>
|
||||
<div class="right">
|
||||
<a name="beijing" href="http://bj.${domain}${root}/">北京</a>
|
||||
<a name="tianjin" href="http://tj.${domain}${root}/">天津</a>
|
||||
<a name="shenyang" href="http://sy.${domain}${root}/">沈阳</a>
|
||||
<a name="shijiazhuang" href="http://sjz.${domain}${root}/">石家庄</a>
|
||||
<a name="dalian" href="http://dl.${domain}${root}/">大连</a>
|
||||
<a name="taiyuan" href="http://ty.${domain}${root}/">太原</a>
|
||||
<a name="changchun" href="http://changchun.${domain}${root}/">长春</a>
|
||||
<a name="haerbin" href="http://haerbin.${domain}${root}/">哈尔滨</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%8C%97%E4%BA%AC">北京</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%A4%A9%E6%B4%A5">天津</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%B2%88%E9%98%B3">沈阳</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E7%9F%B3%E5%AE%B6%E5%BA%84">石家庄</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%A4%A7%E8%BF%9E">大连</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%A4%AA%E5%8E%9F">太原</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%95%BF%E6%98%A5">长春</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%93%88%E5%B0%94%E6%BB%A8">哈尔滨</a>
|
||||
</div>
|
||||
</li>
|
||||
<li>
|
||||
<div class="left">华南华中</div>
|
||||
<div class="right">
|
||||
<a name="guangzhou" href="http://gz.${domain}${root}/">广州</a>
|
||||
<a name="shenzhen" href="http://sz.${domain}${root}/">深圳</a>
|
||||
<a name="wuhan" href="http://wh.${domain}${root}/">武汉</a>
|
||||
<a name="sanya" href="http://sanya.${domain}${root}/">三亚</a>
|
||||
<a name="fuzhou" href="http://fz.${domain}${root}/">福州</a>
|
||||
<a name="changsha" href="http://cs.${domain}${root}/">长沙</a>
|
||||
<a name="xiamen" href="http://xm.${domain}${root}/">厦门</a>
|
||||
<a name="zhengzhou" href="http://zz.${domain}${root}/">郑州</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%B9%BF%E5%B7%9E">广州</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%B7%B1%E5%9C%B3">深圳</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%AD%A6%E6%B1%89">武汉</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E4%B8%89%E4%BA%9A">三亚</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E7%A6%8F%E5%B7%9E">福州</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%95%BF%E6%B2%99">长沙</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%8E%A6%E9%97%A8">厦门</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%83%91%E5%B7%9E">郑州</a>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
<li>
|
||||
<div class="left">西部城市</div>
|
||||
<div class="right">
|
||||
<a name="chengdu" href="http://cd.${domain}${root}/">成都</a>
|
||||
<a name="xian" href="http://xa.${domain}${root}/">西安</a>
|
||||
<a name="chongqing" href="http://cq.${domain}${root}/">重庆</a>
|
||||
<a name="guiyang" href="http://guiyang.${domain}${root}/">贵阳</a>
|
||||
<a name="kunming" href="http://kunming.${domain}${root}/">昆明</a>
|
||||
<a name="yinchuan" href="http://yinchuan.${domain}${root}/">银川</a>
|
||||
<a name="lanzhou" href="http://lanzhou.${domain}${root}/">兰州</a>
|
||||
<a name="wulumuqi" href="http://xj.${domain}${root}/">乌鲁木齐</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%88%90%E9%83%BD">成都</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E8%A5%BF%E5%AE%89">西安</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%87%8D%E5%BA%86">重庆</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E8%B4%B5%E9%98%B3">贵阳</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E6%98%86%E6%98%8E">昆明</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E9%93%B6%E5%B7%9D">银川</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E5%85%B0%E5%B7%9E">兰州</a>
|
||||
<a name="shanghai" href="http://geluke.fangjia.com/map/index?cityName=%E4%B9%8C%E9%B2%81%E6%9C%A8%E9%BD%90">乌鲁木齐</a>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
<li><div class="left"></div><div class="right"><span class="more"><a href="http://www.${domain}${root}/">更多城市>></a></span></div></li>
|
||||
</ul>
|
||||
</div>
|
||||
<#--地图城市切换 end-->
|
||||
@@ -28,7 +28,7 @@ $(document).ready(function(){
|
||||
$("#_h_logout").click(function(){
|
||||
jQuery.ajax({
|
||||
type: "get",
|
||||
url: "http://" + location.host + "${root}/user/logout",
|
||||
url: "http://geluke.fangjia.com/user/logout",
|
||||
dataType:'json',
|
||||
success: function(msg){
|
||||
location.href = '${cityUrl}';
|
||||
|
||||
@@ -200,27 +200,9 @@
|
||||
<script type="text/javascript">
|
||||
$(".tab-pane a, .hotcity a").click(function(){
|
||||
var cityName = this.innerHTML;
|
||||
// var userName = '${loginUserName!}';
|
||||
var name = this.name;
|
||||
|
||||
// if(cityName!='上海'&&userName=='xianlixiong'&&cityName!='北京'){
|
||||
// cityName = '上海';
|
||||
// name = 'sh';
|
||||
// }
|
||||
// if(cityName!='上海'&&userName=='pingping'){
|
||||
// cityName = '上海';
|
||||
// name = 'sh';
|
||||
// }
|
||||
// if((userName=='Angela'||userName=='jialun')&&cityName!='上海'&&cityName!='昆山'){
|
||||
// cityName = '上海';
|
||||
// name = 'sh';
|
||||
// }
|
||||
// $(".cityWidth").text(cityName);
|
||||
|
||||
var url = 'http://'+name+'.${domain}${root}/map/index?cityName='+encodeURIComponent(cityName)+"&t=" + new Date().getTime()
|
||||
var url = 'http://geluke.fangjia.com/map/index?cityName='+encodeURIComponent(cityName)+"&t=" + new Date().getTime()
|
||||
window.location.href = url;
|
||||
|
||||
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user